diff --git a/docker/rocm.Dockerfile b/docker/rocm.Dockerfile index 3682a5264a..50cd8b0429 100644 --- a/docker/rocm.Dockerfile +++ b/docker/rocm.Dockerfile @@ -586,6 +586,12 @@ RUN git clone ${AITER_REPO} \ git revert --no-edit --no-commit e708f6c15; \ fi +# ROCm/aiter#5195's .co with a mask assembled in between the q rows sharing a +# tile, which the grouped dsv4 decode needs; bit-identical at max_seqlen_q=1. +COPY --from=local_src \ + /src/python/sglang/kernels/ops/attention/dsv4/asm/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co \ + /sgl-workspace/aiter/hsa/gfx950/mla_v4/ + # The pinned AITER revision uses std::optional in topk_per_row_kernels.cu without # including : ROCm/aiter#4702 removed the torch headers that previously # supplied it transitively. ROCm 7.0 therefore fails module_top_k_per_row, while diff --git a/python/sglang/kernels/ops/attention/dsv4/asm/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co b/python/sglang/kernels/ops/attention/dsv4/asm/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co new file mode 100755 index 0000000000..410af4a1e3 Binary files /dev/null and b/python/sglang/kernels/ops/attention/dsv4/asm/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co differ diff --git a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/grouped_verify_streams.py b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/grouped_verify_streams.py new file mode 100644 index 0000000000..faf66500bd --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/grouped_verify_streams.py @@ -0,0 +1,139 @@ +""" +Index streams for the grouped target-verify decode. + +The per-token HCA stream is [swa][compressed tail], one per draft, so a +request's kv is read once per draft. The asm kernel can take four drafts +in one tile, but only if the stream is laid out the way its masking +already works: + +[compressed tail][swa] +""" + +from typing import Tuple + +import torch +import triton +import triton.language as tl + +# asm v4 only support specified max_seqlen_q +_ALLOWED_BLOCK_Q = (1, 2, 4) + + +@triton.jit +def _grouped_stream_kernel( + out_ptr, # [*] int32 + indptr_ptr, # [G+1] int32 + slot_ptr, # [G] int32, state slot of the group's request + pos_ptr, # [G] int32, position of the group's LAST draft + tail_len_ptr, # [G] int32, compressed entries, all drafts see them + tail_page_ptr, # [G, TAIL_W] int32 + ring_stride, + swa_pages, + TAIL_W: tl.constexpr, + TAIL_B: tl.constexpr, + WIN_B: tl.constexpr, +): + g = tl.program_id(0) + base = tl.load(indptr_ptr + g) + n_tail = tl.load(tail_len_ptr + g).to(tl.int32) + + # The tail goes first. Every draft in the group sees all of it. + for off in tl.range(0, TAIL_W, TAIL_B): + j = off + tl.arange(0, TAIL_B) + m = j < n_tail + jc = tl.minimum(j, TAIL_W - 1) + pi = tl.load(tail_page_ptr + g * TAIL_W + jc, mask=m, other=-1).to(tl.int32) + tl.store(out_ptr + base + j, tl.where(pi >= 0, pi + swa_pages, -1), mask=m) + + # Then the swa from this group's first draft to its last. + slot = tl.load(slot_ptr + g) + pos = tl.load(pos_ptr + g) + n = tl.load(indptr_ptr + g + 1) - base - n_tail + i = tl.arange(0, WIN_B) + m = i < n + abs_pos = pos - n + 1 + i + val = slot * ring_stride + abs_pos % ring_stride + tl.store(out_ptr + base + n_tail + i, val, mask=m) + + +def _group_index(R, num_draft, block_q, device): + """ + Per-group gather index and group size, built on the device. + + With R=3, num_draft=7, block_q=4 the six groups come out as + + last_token = [3, 6, 10, 13, 17, 20] + group_size = [4, 3, 4, 3, 4, 3] + """ + n_full = num_draft // block_q + remainder = num_draft % block_q + n_groups = n_full + (1 if remainder else 0) + ends = ( + torch.arange(1, n_groups + 1, dtype=torch.int64, device=device) * block_q + ).clamp(max=num_draft) + base = torch.arange(R, dtype=torch.int64, device=device) * num_draft + last_token = (base[:, None] + (ends - 1)).reshape(-1) + group_size = torch.diff(ends, prepend=ends.new_zeros(1)) + return last_token, group_size.to(torch.int32).repeat(R) + + +def build_grouped_verify_streams( + *, + state_slot: torch.Tensor, # [N] int, per query token + positions: torch.Tensor, # [N] int, per query token + hca_len: torch.Tensor, # [N] int, compressed entries per token + hca_page_indices: torch.Tensor, # [N, Wc] int32 + win: int, + ring_stride: int, + swa_pages: int, + num_draft: int, + block_q: int = 4, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Returns (indices, kv_indptr, qo_indptr) for the grouped call. + """ + assert block_q in _ALLOWED_BLOCK_Q, ( + f"block_q={block_q} is not one of {_ALLOWED_BLOCK_Q}; aiter's v4 " + "dispatcher would not route it to the tile this path needs" + ) + dev = state_slot.device + N = state_slot.shape[0] + assert N % num_draft == 0, f"{N} tokens is not a whole number of drafts" + R = N // num_draft + last_token, group_size = _group_index(R, num_draft, block_q, dev) + G = last_token.numel() + + slot_g = state_slot.to(torch.int32)[last_token].contiguous() + pos_g = positions.to(torch.int32)[last_token].contiguous() + tail_g = hca_len.to(torch.int32)[last_token].contiguous() + page_g = hca_page_indices[last_token].contiguous() + + lens = tail_g + torch.minimum(pos_g + 1, win + group_size - 1) + kv_indptr = torch.zeros((G + 1,), dtype=torch.int32, device=dev) + kv_indptr[1:] = lens.cumsum(0).to(torch.int32) + + Wc = page_g.shape[1] + # Sized to the worst case rather than to kv_indptr[-1], which would need a + # device read and so could not be captured in a graph. The streams are + # packed by kv_indptr, so the slack past the last one is never touched. + indices = torch.empty( + (G * (Wc + win + block_q - 1),), dtype=torch.int32, device=dev + ) + _grouped_stream_kernel[(G,)]( + indices, + kv_indptr, + slot_g, + pos_g, + tail_g, + page_g, + ring_stride, + swa_pages, + TAIL_W=max(Wc, 1), + TAIL_B=min(triton.next_power_of_2(max(Wc, 1)), 1024), + WIN_B=triton.next_power_of_2(win + block_q - 1), + num_warps=4, + ) + + qo_indptr = torch.zeros((G + 1,), dtype=torch.int32, device=dev) + qo_indptr[1:] = group_size.cumsum(0).to(torch.int32) + return indices, kv_indptr, qo_indptr diff --git a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py index 426bb8bb6b..5b60c02202 100644 --- a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py +++ b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py @@ -293,6 +293,7 @@ def decode_fp8_2buff( attn_sink: torch.Tensor, # [H] fp32 v_head_dim: int, qo_indptr: Optional[torch.Tensor] = None, + max_seqlen_q: int = 1, num_kv_splits: Optional[int] = None, compress_ratio: Optional[int] = None, ) -> torch.Tensor: @@ -310,6 +311,9 @@ def decode_fp8_2buff( ``compress_ratio`` names the stream (0 SWA, 4 CSA, 128 HCA) so the split count past 40 tokens can fit the stream; None keeps the fixed 4. + + ``max_seqlen_q`` above 1 puts several q rows in one tile, which the v4 + dispatcher serves for gqa=16 at 1, 2 and 4 off the same 64-row tile. """ from aiter.mla import mla_decode_fwd_v4_nm @@ -339,24 +343,21 @@ def decode_fp8_2buff( if qo_indptr is None: qo_indptr = decode_qo_indptr(T, q.device) - # num_seqs comes from qo_indptr.numel()-1 and the kernel writes - # num_seqs * max_seqlen_q rows into `out`, so both have to be sized off q's - # own T. A qo_indptr built from a padded token count writes past `out`. - assert qo_indptr.shape[0] >= T + 1, ( - f"qo_indptr holds {qo_indptr.shape[0]} entries, kernel reads {T + 1}" + n_seq = qo_indptr.shape[0] - 1 + assert kv_indptr.shape[0] >= n_seq + 1, ( + f"kv_indptr holds {kv_indptr.shape[0]} entries, kernel reads {n_seq + 1}" ) - assert kv_indptr.shape[0] >= T + 1, ( - f"kv_indptr holds {kv_indptr.shape[0]} entries, kernel reads {T + 1}" - ) - qo_indptr = qo_indptr[: T + 1] rows = unified_kv.shape[0] - out = q_rope.new_empty((T, H, v_head_dim)) + out = q_rope.new_empty((n_seq * max_seqlen_q, H, v_head_dim)) # Left None, the wrapper's occupancy heuristic picks it, folds the cross-split # merge back into `out`, and leaves the final bf16 there whether or not it # split. Pinning it to 1 costs 6.9x at bs=1 kv=2048. - if num_kv_splits is None and T > _DECODE_SPLIT_TAIL_MIN_TOKENS: - num_kv_splits = _decode_fp8_tail_splits(T, compress_ratio) + # Sequences, not tokens: the wave-fitting rule counts the workgroups the + # kernel launches, which is n_seq x splits. They are the same number on the + # per-token path and differ once q rows share a tile. + if num_kv_splits is None and n_seq > _DECODE_SPLIT_TAIL_MIN_TOKENS: + num_kv_splits = _decode_fp8_tail_splits(n_seq, compress_ratio) mla_decode_fwd_v4_nm( q, q_rope, @@ -366,7 +367,7 @@ def decode_fp8_2buff( qo_indptr, kv_indptr, kv_indices, - 1, # max_seqlen_q; qo_indptr is per-token so every sequence is one token + max_seqlen_q, sink=attn_sink, num_kv_splits=num_kv_splits, ) @@ -374,7 +375,7 @@ def decode_fp8_2buff( # ReqToTokenPool reserves, so the builders can't emit a zero-length one, and # the compare + masked_fill_ was costing a launch per layer for it. One would # come back NaN now (all-sink denominator); the guard UT pins that. - return out + return out[:T] @triton.jit diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index 06ef59b6bd..b9b05beea3 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -3,6 +3,7 @@ from __future__ import annotations import enum import functools import logging +import os from dataclasses import dataclass, field from typing import ( TYPE_CHECKING, @@ -104,6 +105,14 @@ class UnifiedKvMetadata: csa_indices: Optional[torch.Tensor] = None csa_indptr: Optional[torch.Tensor] = None + # Grouped target-verify streams for the asm decode: one per (request, + # draft group), laid out [compressed tail][that group's window slice] so + # the kernel's own causal bound and band mask land on each draft's real + # cutoff. + gasm_indices: Optional[torch.Tensor] = None + gasm_kv_indptr: Optional[torch.Tensor] = None + gasm_qo_indptr: Optional[torch.Tensor] = None + # prefill/extend per-token mapping pf_state_slot: Optional[torch.Tensor] = None pf_chunk_start: Optional[torch.Tensor] = None @@ -131,6 +140,9 @@ class UnifiedKvMetadata: "swa_indptr", "hca_indices", "hca_indptr", + "gasm_indices", + "gasm_kv_indptr", + "gasm_qo_indptr", "csa_indices", "csa_indptr", "pf_state_slot", @@ -160,6 +172,9 @@ class UnifiedKvMetadata: "swa_indptr", "hca_indices", "hca_indptr", + "gasm_indices", + "gasm_kv_indptr", + "gasm_qo_indptr", "csa_indices", "csa_indptr", "pf_state_slot", @@ -511,6 +526,24 @@ class DSV4RawDecodeMetadata: self.out_cache_loc.copy_(other.out_cache_loc) +_GROUPED_ASM_BLOCK_Q = 4 + +# aiter serves gqa*msq off one 64-row q tile and only whitelists msq above 1 +# at gqa=16 -- the TP-only shape. DP attention gives a rank all 128 heads, so +# the tile is full of heads with no room to group, and none is needed either: +# one kv read already feeds 128 heads rather than 16. An unsupported pair does +# not downgrade, it fails aiter's kernel lookup outright. +_GROUPED_ASM_GQA = 16 +_GROUPED_ASM_MIN_REQS = 16 + + +def _grouped_asm_enabled() -> bool: + """ + Grouped target-verify decode through the asm kernel, off by default. + """ + return os.environ.get("SGLANG_DSV4_GROUPED_ASM", "0") == "1" + + class _GraphBucket(enum.Enum): DECODE_OR_IDLE = "decode_or_idle" TARGET_VERIFY = "target_verify" @@ -714,7 +747,9 @@ class DeepseekV4HipRadixBackend( # per-token (num_draft*bs -> bs) req-slot map produced by the prefill # expansion above. self._attach_unified_kv_decode_streams( - core_attn_metadata, req_pool_indices_repeated + core_attn_metadata, + req_pool_indices_repeated, + num_draft=self.target_verify_num_draft_tokens, ) indexer_metadata = ( self.init_forward_metadata_indexer(core_attn_metadata) @@ -923,7 +958,9 @@ class DeepseekV4HipRadixBackend( num_draft_tokens * bs, ) self._attach_unified_kv_decode_streams( - core_attn_metadata, req_pool_indices_repeated + core_attn_metadata, + req_pool_indices_repeated, + num_draft=self.target_verify_num_draft_tokens, ) indexer_metadata = ( self.init_forward_metadata_indexer(core_attn_metadata) @@ -1469,7 +1506,10 @@ class DeepseekV4HipRadixBackend( self.forward_metadata = current_raw def _attach_unified_kv_decode_streams( - self, core: DSV4AttnMetadata, state_slot: torch.Tensor + self, + core: DSV4AttnMetadata, + state_slot: torch.Tensor, + num_draft: int = 0, ) -> None: # state_slot maps each query token to its request slot; # target-verify repeats request slots for the draft tokens. @@ -1521,6 +1561,35 @@ class DeepseekV4HipRadixBackend( ring_stride=pool.unified_swa_ring_size, swa_pages=pool.unified_swa_pages, ) + core.unified.gasm_indices = None + core.unified.gasm_kv_indptr = None + core.unified.gasm_qo_indptr = None + if ( + _grouped_asm_enabled() + and num_draft > 1 + and N % num_draft == 0 + and N // num_draft >= _GROUPED_ASM_MIN_REQS + ): + from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import ( + grouped_verify_streams, + ) + + ( + core.unified.gasm_indices, + core.unified.gasm_kv_indptr, + core.unified.gasm_qo_indptr, + ) = grouped_verify_streams.build_grouped_verify_streams( + state_slot=state_slot, + positions=core.positions_casual, + hca_len=hca_len, + hca_page_indices=hca_page_indices, + win=pool.unified_swa_window, + ring_stride=pool.unified_swa_ring_size, + swa_pages=pool.unified_swa_pages, + num_draft=num_draft, + block_q=_GROUPED_ASM_BLOCK_Q, + ) + # SWA ring write target, same value for every layer this forward. req_slot = state_slot.to(torch.int64) core.unified.swa_loc = ( @@ -1691,6 +1760,25 @@ class DeepseekV4HipRadixBackend( "the v4 nm asm kernel hardcodes 1/sqrt(512), this backend is " f"at {self.softmax_scale}" ) + gasm = ( + unified_metadata.gasm_indices + if compress_ratio == 128 and q.shape[1] == _GROUPED_ASM_GQA + else None + ) + if gasm is not None: + return runtime.decode_fp8_2buff( + q=q, + q_rope=q_rope, + unified_kv=unified, + unified_kv_rope=pool.get_unified_kv_rope(layer_id), + kv_indices=gasm, + kv_indptr=unified_metadata.gasm_kv_indptr, + qo_indptr=unified_metadata.gasm_qo_indptr, + attn_sink=attn_sink, + v_head_dim=layer.v_head_dim, + max_seqlen_q=_GROUPED_ASM_BLOCK_Q, + compress_ratio=compress_ratio, + ) return runtime.decode_fp8_2buff( q=q, q_rope=q_rope,