diff --git a/python/sglang/kernels/ops/attention/minimax_sparse/decode/indexer_cp.py b/python/sglang/kernels/ops/attention/minimax_sparse/decode/indexer_cp.py index 812755eaae..dff09b01cc 100644 --- a/python/sglang/kernels/ops/attention/minimax_sparse/decode/indexer_cp.py +++ b/python/sglang/kernels/ops/attention/minimax_sparse/decode/indexer_cp.py @@ -9,6 +9,9 @@ import torch import triton import triton.language as tl +# Rows per scoring tile; unrelated to the selector's top-k, which is also 16. +TILE_ROWS = 16 + @triton.jit def _score_shard( @@ -92,6 +95,102 @@ def _score_shard( ) +@triton.jit +def _score_shard_packed( + Q, + K, + ReqToToken, + Slots, + Lengths, + Scores, + BATCH: tl.constexpr, + WORLD: tl.constexpr, + RANK: tl.constexpr, + PACK: tl.constexpr, + TILE: tl.constexpr, + MAX_BLOCKS: tl.constexpr, + LOCAL_BLOCKS: tl.constexpr, + Q_HEAD_STRIDE: tl.constexpr, + Q_ROW_STRIDE: tl.constexpr, + K_SLOT_STRIDE: tl.constexpr, + K_DIM_STRIDE: tl.constexpr, + TABLE_STRIDE: tl.constexpr, + TABLE_ROWS: tl.constexpr, + CACHE_SLOTS: tl.constexpr, + BLOCKS_PER_CHUNK: tl.constexpr, + INIT_BLOCKS: tl.constexpr, + LOCAL_KEEP: tl.constexpr, + sm_scale, + k_scale, +): + # tile row i is (draft row i // WORLD, head i % WORLD): one K read per request + group, chunk = tl.program_id(0), tl.program_id(1) + tile = tl.arange(0, TILE) + head, sub = tile % WORLD, tile // WORLD + in_tile = tile < PACK * WORLD + rows = group * PACK + sub + dims = tl.arange(0, 128) + positions = tl.arange(0, 128) + lengths = tl.load(Lengths + rows, mask=in_tile, other=0) + group_len = tl.max(lengths, 0) + # caller contract: a group's PACK rows share one request, so one slot addresses K + request = tl.load(Slots + group * PACK).to(tl.int64) + # Match the native scorer's request/slot addressing, including padding. + request = (request + CACHE_SLOTS) % CACHE_SLOTS + active = (lengths > 0) & (request >= 0) & (request < TABLE_ROWS) & in_tile + num_blocks = tl.cdiv(lengths, 128) + local_start = tl.maximum(0, num_blocks - LOCAL_KEEP) + q = tl.load( + Q + + head[:, None] * Q_HEAD_STRIDE + + rows[:, None] * Q_ROW_STRIDE + + dims[None, :], + mask=in_tile[:, None], + other=0, + ) + start = chunk * BLOCKS_PER_CHUNK + end = tl.minimum(start + BLOCKS_PER_CHUNK, LOCAL_BLOCKS) + for local in range(start, end): + block = local * WORLD + RANK + valid = active & (block < MAX_BLOCKS) & (block < num_blocks) + # At <=top-k blocks the native selector emits every block without + # reading scores. In particular, graph padding need not read K. + score = tl.full((TILE,), 0.0, tl.float32) + # one branch per group: its longest row decides whether K is read; + # a padding slot past the table must not index ReqToToken + group_blocks = tl.cdiv(group_len, 128) + if (request < TABLE_ROWS) & (block < group_blocks) & (group_blocks > 16): + pos = block * 128 + positions + token_valid = pos < group_len + slots = tl.load( + ReqToToken + request * TABLE_STRIDE + pos, + mask=token_valid, + other=0, + ).to(tl.int64) + slots = (slots + CACHE_SLOTS) % CACHE_SLOTS + k = tl.load( + K + dims[:, None] * K_DIM_STRIDE + slots[None, :] * K_SLOT_STRIDE, + mask=token_valid[None, :], + other=0.0, + ).to(q.dtype) + # same dot orientation and scale order as _score_shard, so scores match exactly + dot = tl.dot(q, k) * (sm_scale * 1.4426950409 * k_scale) + dot = tl.where(pos[None, :] < lengths[:, None], dot, float("-inf")) + score = tl.max(dot, 1) + score = tl.where( + block >= local_start, + 1e29, + tl.where(block < INIT_BLOCKS, 1e30, score), + ) + score = tl.where(valid, score, float("-inf")) + # Every slot is written on every replay, including empty shards. + tl.store( + Scores + (head * BATCH + rows) * LOCAL_BLOCKS + local, + score, + mask=in_tile, + ) + + @triton.jit def _pack(score, block, valid): # Same fp32 total order as ROCm TopKTrait::pack_score_id. 16 ID bits @@ -190,14 +289,56 @@ def score_local_blocks( local_blocks, sm_scale, k_scale, + packed_queries=1, ): - """Score [world,batch,128] queries through SGLang's token-slot mapping.""" + """Score [world,batch,128] queries through SGLang's token-slot mapping. + + With packed_queries > 1, each group of that many consecutive rows belongs to one request + and reads each K block once for the whole group. + """ world, batch, _ = gathered_q.shape blocks = triton.cdiv(max_seqlen, 128) local = triton.cdiv(blocks, world) scores = torch.empty( (world, batch, local), dtype=torch.float32, device=gathered_q.device ) + if ( + packed_queries > 1 + and batch % packed_queries == 0 + and packed_queries * world <= TILE_ROWS + ): + groups = batch // packed_queries + chunks = min(local, max(1, min(256, 4096 // max(groups, 1)))) + _score_shard_packed[(groups, chunks)]( + gathered_q, + k_cache, + req_to_token, + slot_ids, + seq_lens, + scores, + BATCH=batch, + WORLD=world, + RANK=rank, + PACK=packed_queries, + TILE=TILE_ROWS, + MAX_BLOCKS=blocks, + LOCAL_BLOCKS=local, + Q_HEAD_STRIDE=gathered_q.stride(0), + Q_ROW_STRIDE=gathered_q.stride(1), + K_SLOT_STRIDE=k_cache.stride(0), + K_DIM_STRIDE=k_cache.stride(2), + TABLE_STRIDE=req_to_token.stride(0), + TABLE_ROWS=req_to_token.shape[0], + CACHE_SLOTS=k_cache.shape[0], + BLOCKS_PER_CHUNK=triton.cdiv(local, chunks), + INIT_BLOCKS=init_blocks, + LOCAL_KEEP=local_blocks, + sm_scale=sm_scale, + k_scale=k_scale, + num_warps=4, + num_stages=1, + ) + return scores chunks = min(local, max(1, min(256, 4096 // max(batch, 1)))) if batch: _score_shard[(batch, chunks)]( diff --git a/python/sglang/srt/layers/attention/minimax_sparse_backend.py b/python/sglang/srt/layers/attention/minimax_sparse_backend.py index a849510da1..b16e5039f4 100644 --- a/python/sglang/srt/layers/attention/minimax_sparse_backend.py +++ b/python/sglang/srt/layers/attention/minimax_sparse_backend.py @@ -1843,6 +1843,12 @@ class MiniMaxSparseAttnBackend(AttentionBackend): topk_out=_topk_buf if _want_topk else None, hisparse_swap_in_fn=hisparse_swap_in_fn, indexer_cp=self.indexer_cp, + # the verify funnel lays out each request's draft rows consecutively + packed_queries=( + self.speculative_num_draft_tokens + if forward_batch.forward_mode.is_target_verify() + else 1 + ), ) return ( None if idx_o is None else idx_o.reshape(q.shape[0], -1).contiguous(), diff --git a/python/sglang/srt/layers/attention/minimax_sparse_ops/indexer_cp.py b/python/sglang/srt/layers/attention/minimax_sparse_ops/indexer_cp.py index a9fb0f0841..3aa15e0fca 100644 --- a/python/sglang/srt/layers/attention/minimax_sparse_ops/indexer_cp.py +++ b/python/sglang/srt/layers/attention/minimax_sparse_ops/indexer_cp.py @@ -162,6 +162,7 @@ class MiniMaxIndexerCP: sm_scale=None, q_scale=None, k_scale=None, + packed_queries=1, ): from sglang.kernels.ops.attention.minimax_sparse.decode.indexer_cp import ( merge_candidates, @@ -189,6 +190,7 @@ class MiniMaxIndexerCP: local_blocks, sm_scale, 1.0 if k_scale is None else k_scale, + packed_queries=packed_queries, ) keys = select_local_candidates(scores, seq_lens, self.rank, max_seqlen) received = torch.empty((4, 4, batch, 16), dtype=torch.int64, device=q.device) diff --git a/python/sglang/srt/layers/attention/minimax_sparse_ops/minimax_sparse.py b/python/sglang/srt/layers/attention/minimax_sparse_ops/minimax_sparse.py index 018a38e329..5900952999 100644 --- a/python/sglang/srt/layers/attention/minimax_sparse_ops/minimax_sparse.py +++ b/python/sglang/srt/layers/attention/minimax_sparse_ops/minimax_sparse.py @@ -327,6 +327,7 @@ def minimax_sparse_decode( topk_out: Optional[torch.Tensor] = None, hisparse_swap_in_fn: Optional[Callable] = None, indexer_cp=None, + packed_queries: int = 1, # consecutive rows sharing one request (EAGLE verify) ) -> Tuple[torch.Tensor, torch.Tensor]: # Index top-k sharing for DECODE. A group's source layer passes ``topk_out`` # (a persistent buffer) and publishes its reduced top-k there; the group's @@ -362,6 +363,7 @@ def minimax_sparse_decode( idx_sm_scale, idx_q_scale, idx_k_scale, + packed_queries=packed_queries, ) _skip_reduce = True else: diff --git a/test/registered/amd/test_minimax_indexer_cp.py b/test/registered/amd/test_minimax_indexer_cp.py index cd2b9a7453..91e8b8d3a6 100644 --- a/test/registered/amd/test_minimax_indexer_cp.py +++ b/test/registered/amd/test_minimax_indexer_cp.py @@ -4,30 +4,46 @@ import unittest import torch +from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci register_amd_ci(est_time=20, suite="stage-b-test-1-gpu-small-amd-mi35x") BLOCK, TOPK, HEADS, DIM = 128, 16, 4, 128 INIT_BLOCKS, LOCAL_BLOCKS = 1, 2 +DRAFT_TOKENS = 4 def _inputs(batch, max_len, dtype, device): torch.manual_seed(20260928) padded_len = (max_len + BLOCK - 1) // BLOCK * BLOCK - nslots = batch * padded_len + # slots are never the row or group index, so addressing K by index reads the wrong keys + table_rows = 3 * batch + nslots = table_rows * padded_len # Physical pages shuffled independently of logical block ownership, so a rank # reading blocks r, r+4, ... cannot accidentally read contiguous memory. pages = torch.randperm(nslots // 16, device=device, dtype=torch.int32) table = ( (pages[:, None] * 16 + torch.arange(16, device=device)) - .reshape(batch, padded_len) + .reshape(table_rows, padded_len) .to(torch.int32) ) cache = torch.randn((nslots, 1, DIM), device=device, dtype=torch.bfloat16).to(dtype) q = torch.randn((batch, HEADS, DIM), device=device, dtype=torch.bfloat16) lengths = torch.full((batch,), max_len, dtype=torch.int64, device=device) - slots = torch.arange(batch, dtype=torch.int64, device=device) + slots = (batch + 1 + 2 * torch.arange(batch, device=device)).flip(0) + return q, cache, table, slots, lengths + + +def _verify_rows(reqs, prefix, dtype, device): + """Chain verify as the backend funnel lays it out: each request's DRAFT_TOKENS draft rows + consecutively, row i seeing prefix + i + 1 tokens.""" + _, cache, table, slots, _ = _inputs(reqs, prefix + DRAFT_TOKENS, dtype, device) + slots = slots.repeat_interleave(DRAFT_TOKENS) + lengths = (prefix + torch.arange(1, DRAFT_TOKENS + 1, device=device)).repeat(reqs) + q = torch.randn( + (reqs * DRAFT_TOKENS, HEADS, DIM), device=device, dtype=torch.bfloat16 + ) return q, cache, table, slots, lengths @@ -54,7 +70,7 @@ def _native_topk(q_head, cache, table, slots, lengths, max_len, k_scale): )[1] -def _cp_topk(q, cache, table, slots, lengths, max_len, k_scale): +def _cp_topk(q, cache, table, slots, lengths, max_len, k_scale, packed_queries=1): """Every rank's shard, computed in one process; the all-gather is the only part a four-rank run adds, and it is the runtime's collective, not this kernel.""" from sglang.kernels.ops.attention.minimax_sparse.decode.indexer_cp import ( @@ -78,13 +94,16 @@ def _cp_topk(q, cache, table, slots, lengths, max_len, k_scale): LOCAL_BLOCKS, DIM**-0.5, k_scale, + packed_queries=packed_queries, ) keys.append(select_local_candidates(scores, lengths, rank, max_len, TOPK)) gathered_keys = torch.stack(keys, dim=0) return [merge_candidates(gathered_keys, head) for head in range(HEADS)] -@unittest.skipUnless(torch.version.hip, "MiniMax indexer CP is gfx950-only") +@unittest.skipUnless( + is_hip() and is_gfx95_supported(), "MiniMax indexer CP is gfx950-only" +) class TestMiniMaxIndexerCP(unittest.TestCase): def test_sharded_selection_matches_the_native_selector(self): device = torch.device("cuda") @@ -113,6 +132,40 @@ class TestMiniMaxIndexerCP(unittest.TestCase): f"head {head}: {(cp_heads[head] != native).sum()} IDs differ", ) + def test_packed_verify_rows_match_the_native_selector(self): + """Packing a request's draft rows into one tile must select what scoring each + row alone does, including rows whose lengths straddle a block boundary.""" + device = torch.device("cuda") + prefix = 255 * BLOCK + 126 # rows end at 32767..32770, across block 256 + max_len = prefix + DRAFT_TOKENS + for dtype in (torch.bfloat16, torch.float8_e4m3fnuz): + with self.subTest(dtype=dtype): + q, cache, table, slots, lengths = _verify_rows(4, prefix, dtype, device) + packed = _cp_topk( + q, + cache, + table, + slots, + lengths, + max_len, + 1.0, + packed_queries=DRAFT_TOKENS, + ) + for head in range(HEADS): + native = _native_topk( + q[:, head : head + 1], + cache, + table, + slots, + lengths, + max_len, + 1.0, + ) + self.assertTrue( + torch.equal(packed[head], native), + f"head {head}: {(packed[head] != native).sum()} IDs differ", + ) + if __name__ == "__main__": unittest.main()