[AMD][V4.1][*/N] OPUS sparse prefill on gfx950 through layout conversion (#42017)

Co-authored-by: HAI <hixiao@gmail.com>
Co-authored-by: thomawan <thomawan@amd.com>
This commit is contained in:
Xinyi Song
2026-10-01 03:21:25 -07:00
committed by GitHub
co-authored by HAI thomawan
parent f45c004e18
commit 3c4e218779
4 changed files with 497 additions and 3 deletions
@@ -0,0 +1,101 @@
"""DeepSeek-V4 sparse prefill attention on gfx950 through aiter's OPUS kernel.
OPUS attends over two bf16 row sources, each addressed by a CSR index list. The
caller dequantizes rows into bf16 workspaces and builds padded per-query lists
(``SparsePrefillChunkCache``); this module turns those lists into CSR and runs
the kernel.
"""
from typing import Optional, Tuple
import torch
import triton
import triton.language as tl
# (int32 indices, int32 indptr [T + 1])
Csr = Tuple[torch.Tensor, torch.Tensor]
@triton.jit
def _combined_to_csr_kernel(
indices_ptr,
indices_stride,
lens_ptr,
indptr_ptr,
out_ptr,
row_start,
row_end,
BLOCK: tl.constexpr,
):
# OPUS reads every CSR entry, so -1 holes inside the scanned prefix and
# entries outside [row_start, row_end) are dropped here.
row = tl.program_id(0).to(tl.int64)
cols = tl.arange(0, BLOCK)
length = tl.load(lens_ptr + row)
idx = tl.load(
indices_ptr + row * indices_stride + cols, mask=cols < length, other=-1
)
keep = (idx >= row_start) & (idx < row_end)
dst = tl.cumsum(keep.to(tl.int32), axis=0) - 1
start = tl.load(indptr_ptr + row)
tl.store(out_ptr + start + dst, idx - row_start, mask=keep)
def combined_to_csr(
indices: torch.Tensor, lens: torch.Tensor, row_start: int, row_end: int
) -> Csr:
"""int32 [T, W] padded index lists with per-row scanned lengths [T] -> the CSR
(T * W capacity) of their entries in [row_start, row_end), rebased to row_start."""
num_rows, width = indices.shape
device = indices.device
cols = torch.arange(width, device=device, dtype=torch.int32)
keep = (
(cols[None, :] < lens[:, None]) & (indices >= row_start) & (indices < row_end)
)
indptr = torch.zeros(num_rows + 1, dtype=torch.int32, device=device)
torch.cumsum(keep.sum(dim=1, dtype=torch.int32), dim=0, out=indptr[1:])
csr = torch.empty(max(num_rows * width, 1), dtype=torch.int32, device=device)
if num_rows > 0:
_combined_to_csr_kernel[(num_rows,)](
indices,
indices.stride(0),
lens,
indptr,
csr,
row_start,
row_end,
BLOCK=triton.next_power_of_2(width),
)
return csr, indptr
def opus_sparse_prefill(
q: torch.Tensor,
kv: torch.Tensor,
csr: Csr,
attn_sink: torch.Tensor,
softmax_scale: float,
extend_kv: Optional[torch.Tensor] = None,
extend_csr: Optional[Csr] = None,
) -> torch.Tensor:
"""Attention of q [T, H, 512] bf16 over the bf16 rows of kv [R, 512] named by
csr, then those of extend_kv named by extend_csr, with a per-head sink."""
from aiter.ops.pa_sparse_prefill_opus import pa_sparse_prefill_opus
indices, indptr = csr
if extend_kv is None:
# every query reads an empty row of the unused extend source
extend_kv = kv[:1]
extend_csr = (indices[:1], torch.zeros_like(indptr))
extend_indices, extend_indptr = extend_csr
return pa_sparse_prefill_opus(
q.contiguous(),
kv,
indices,
indptr,
extend_kv,
extend_indices,
extend_indptr,
attn_sink,
softmax_scale,
)
+3
View File
@@ -1579,6 +1579,9 @@ class Envs:
SGLANG_DSV41_TORCH_PREFILL_INDEXER = EnvBool(False)
SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False)
SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True)
# gfx950 DeepSeek-V4.1 prefill: attend with aiter's OPUS sparse kernel over a bf16
# dequant of the chunk's SWA and compressed history, instead of the decode kernel.
SGLANG_OPT_HIP_OPUS_SPARSE_PREFILL = EnvBool(False)
# DSpark draft block on the HIP radix backend: build the attention metadata inside the
# draft CUDA graph from the raw inputs instead of eagerly before every replay.
SGLANG_HIP_DSPARK_DRAFT_RAW_METADATA = EnvBool(_default_hip)
@@ -44,10 +44,16 @@ from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import (
from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import (
SOFTMAX_SCALE as SWAPAB_SOFTMAX_SCALE,
)
from sglang.kernels.ops.attention.dsv4.dequant_k_cache import dequantize_k_cache_paged
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
from sglang.kernels.ops.attention.dsv4.metadata_kernel import (
init_compression_metadata as _init_compression_metadata_triton,
)
from sglang.kernels.ops.attention.dsv4.opus_sparse_prefill_hip import (
Csr,
combined_to_csr,
opus_sparse_prefill,
)
from sglang.kernels.ops.attention.dsv4.swapab_gluon_hip import swapab_attention
from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import (
BuildCausalSwaPageIndices,
@@ -81,12 +87,18 @@ from sglang.srt.layers.attention.dsv4.metadata import (
copy_metadata,
maybe_copy_inplace,
)
from sglang.srt.layers.attention.dsv4.sparse_prefill_utils import (
SparsePrefillChunkCache,
SparsePrefillWorkspace,
)
from sglang.srt.layers.attention.hip_flash_mla import (
_apply_inverse_rope,
flash_mla_with_kvcache_entrypoint,
hip_attn_kv_splits,
resolve_hip_flashmla_backend,
)
from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask
from sglang.srt.layers.cp.utils import is_cp_active
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.mem_cache.dsv41_request_window import window_layout
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
@@ -97,7 +109,7 @@ from sglang.srt.runtime_context import (
)
from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc
from sglang.srt.speculative.ragged_verify import resolve_ragged_verify_layout
from sglang.srt.utils import ceil_align
from sglang.srt.utils import ceil_align, is_gfx95_supported
if TYPE_CHECKING:
from sgl_kernel.flash_mla import FlashMLASchedMeta
@@ -130,6 +142,34 @@ def _pad_last_dim(x: T, multiples_of: int = PAGE_INDEX_ALIGNED_SIZE) -> T:
return F.pad(x, pad=(0, target_size - curr_size), mode="constant", value=-1)
# OPUS prefill applies from this many query rows; below it the decode kernel splits KV
# and the dequant is not amortized.
_OPUS_PREFILL_MIN_TOKENS = 1024
# The OPUS workspace holds every request's whole compressed history, dequantized once
# per kv_source layer. Past this many history rows per query row that dequant costs
# more than OPUS saves (gfx950, uniformly random top-k): a ratio-2 source serves fewer
# layers than the ratio-1 one.
_OPUS_PREFILL_MAX_ROWS_PER_TOKEN = {1: 256, 2: 128}
# One ratio's bf16 workspace stays within 1 GiB.
_OPUS_PREFILL_MAX_WORKSPACE_ROWS = 1 << 20
@dataclass
class _OpusPrefillState:
"""OPUS prefill inputs of one forward. A V4.1 ratio-1/2 layer reads the
compressed KV of its kv_source layer and the top-k of its index_source layer,
so each is built once and reused by the layers that follow its source."""
cache: SparsePrefillChunkCache
# the SWA rows every layer lays out alike
swa_csr: Csr
# ratio -> (data_ptr of the compressed cache, its bf16 rows)
compressed_kv: Dict[int, Tuple[int, torch.Tensor]] = field(default_factory=dict)
# ratio -> CSR of the top-k into compressed_kv[ratio]; dropped when the indexer
# rewrites that ratio's top-k
topk_csr: Dict[int, Csr] = field(default_factory=dict)
def _fold_lengths_into_index_lists(
indices: torch.Tensor,
lengths: Optional[torch.Tensor],
@@ -625,11 +665,17 @@ class DSV4AttnMetadata:
self.c0_flashmla_metadata = _create_flashmla_metadata()
self.c4_flashmla_metadata = _create_flashmla_metadata()
self.c128_flashmla_metadata = _create_flashmla_metadata()
# -1 filled like the page indices: identity rows write only their reachable columns,
# and the OPUS prefill reads the raw indices without the top-k lengths
if 1 in self.low_ratios:
self.c1_sparse_raw_indices = torch.empty_like(self.c1_sparse_page_indices)
self.c1_sparse_raw_indices = torch.full_like(
self.c1_sparse_page_indices, -1
)
self.c1_flashmla_metadata = _create_flashmla_metadata()
if 2 in self.low_ratios:
self.c2_sparse_raw_indices = torch.empty_like(self.c2_sparse_page_indices)
self.c2_sparse_raw_indices = torch.full_like(
self.c2_sparse_page_indices, -1
)
self.c2_flashmla_metadata = _create_flashmla_metadata()
@@ -973,6 +1019,12 @@ class DeepseekV4HipRadixBackend(
DSV4RawVerifyMetadata,
DSV4RawDecodeMetadata,
] = None
self.opus_prefill_workspace = SparsePrefillWorkspace(self.device)
self.opus_compressed_workspaces = {
ratio: SparsePrefillWorkspace(self.device) for ratio in (1, 2)
}
# Reset with the metadata: its inputs belong to one prefill forward.
self._opus_prefill_state: Optional[_OpusPrefillState] = None
def _move_to_device(self, x: List[int]) -> torch.Tensor:
pin_tensor = torch.tensor(x, dtype=torch.int32, pin_memory=True)
@@ -1996,6 +2048,7 @@ class DeepseekV4HipRadixBackend(
if self.mtp_enabled and forward_batch.forward_mode.is_idle():
return
self._opus_prefill_state = None
self.forward_metadata = self._build_forward_metadata(forward_batch)
self.init_forward_metadata_in_graph(forward_batch)
self._refresh_fp4_prefill_workspace(forward_batch)
@@ -2210,6 +2263,7 @@ class DeepseekV4HipRadixBackend(
def init_forward_metadata_for_breakable_cuda_graph_capture(
self, forward_batch: ForwardBatch
) -> DSV4Metadata:
self._opus_prefill_state = None
if not self.low_ratios:
self.forward_metadata = self._build_forward_metadata(
forward_batch,
@@ -2253,6 +2307,7 @@ class DeepseekV4HipRadixBackend(
*,
static_forward_batch: Optional[ForwardBatch] = None,
) -> None:
self._opus_prefill_state = None
if not self.low_ratios:
replay_batch = (
static_forward_batch
@@ -2821,6 +2876,169 @@ class DeepseekV4HipRadixBackend(
cache_k=swa_k,
)
def _opus_prefill_applies(
self, q: torch.Tensor, forward_batch: ForwardBatch, compress_ratio: int
) -> bool:
if not envs.SGLANG_OPT_HIP_OPUS_SPARSE_PREFILL.get():
return False
if (
compress_ratio not in (0, 1, 2)
or self.is_draft_worker
or not forward_batch.forward_mode.is_extend_without_speculative()
or q.ndim != 3
or q.shape[0] < _OPUS_PREFILL_MIN_TOKENS
or q.shape[-1] != 512
or q.dtype != torch.bfloat16
or not is_gfx95_supported()
or self.token_to_kv_pool.request_window is not None
or self.forward_metadata.late_layer_tail is not None
or is_cp_active(forward_batch)
or forward_batch.seq_lens_cpu is None
or forward_batch.extend_seq_lens_cpu is None
):
return False
if compress_ratio == 0:
return True
seq_lens = forward_batch.seq_lens_cpu
history_rows = len(seq_lens) * max(int(seq_lens.max()) // compress_ratio, 1)
return history_rows <= min(
_OPUS_PREFILL_MAX_ROWS_PER_TOKEN[compress_ratio] * q.shape[0],
_OPUS_PREFILL_MAX_WORKSPACE_ROWS,
)
def _build_opus_prefill_cache(
self,
forward_batch: ForwardBatch,
core_attn_metadata: DSV4AttnMetadata,
num_tokens: int,
) -> SparsePrefillChunkCache:
seq_lens_cpu = forward_batch.seq_lens_cpu.tolist()
total_swa = sum(
min(int(seq_len), int(extend_len) + SWA_WINDOW - 1)
for seq_len, extend_len in zip(
seq_lens_cpu, forward_batch.extend_seq_lens_cpu, strict=True
)
)
extend_seq_lens = forward_batch.extend_seq_lens.to(torch.int32)
query_pos = core_attn_metadata.seq_lens_casual[:num_tokens] - 1
assert query_pos.shape[0] == num_tokens, (query_pos.shape, num_tokens)
return SparsePrefillChunkCache.build(
seq_lens=forward_batch.seq_lens.to(torch.int32),
extend_seq_lens=extend_seq_lens,
query_lens=extend_seq_lens,
query_pos=query_pos.to(torch.int32),
req_pool_indices=forward_batch.req_pool_indices.to(torch.int32),
req_to_token=self.req_to_token,
full_to_swa=self.token_to_kv_pool.full_to_swa_index_mapping,
swa_window_size=SWA_WINDOW,
swa_page_size=self.token_to_kv_pool.swa_kv_pool.page_size,
num_qo_tokens=num_tokens,
max_seq_len=max(seq_lens_cpu),
total_swa=total_swa,
)
def _forward_prefill_opus(
self,
q: torch.Tensor,
layer_id: int,
compress_ratio: int,
forward_batch: ForwardBatch,
core_attn_metadata: DSV4AttnMetadata,
attn_sink: torch.Tensor,
inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]],
) -> torch.Tensor:
"""Sparse prefill through OPUS over the layer's SWA rows and, for ratios 1 / 2,
the compressed history its top-k selects."""
pool = self.token_to_kv_pool
num_tokens = q.shape[0]
state = self._opus_prefill_state
if state is None or state.cache.num_qo_tokens != num_tokens:
cache = self._build_opus_prefill_cache(
forward_batch, core_attn_metadata, num_tokens
)
swa_csr = combined_to_csr(
cache.c0_combined_indices,
cache.c0_combined_lens,
0,
cache.swa_token_ids.shape[0],
)
state = self._opus_prefill_state = _OpusPrefillState(cache, swa_csr)
cache = state.cache
num_swa_rows = cache.swa_token_ids.shape[0]
swa_rows = self.opus_prefill_workspace.get(num_swa_rows)
dequantize_k_cache_paged(
pool.get_swa_key_buffer_radix(layer_id),
cache.swa_token_ids,
page_size=cache.swa_page_size,
out=swa_rows,
layout=pool.get_swa_key_layout(),
)
swa_kv = swa_rows.view(num_swa_rows, -1)
if compress_ratio == 0:
out = opus_sparse_prefill(
q, swa_kv, state.swa_csr, attn_sink, self.softmax_scale
)
else:
compressed_kv, topk_csr = self._opus_compressed_inputs(
state, layer_id, compress_ratio, core_attn_metadata
)
out = opus_sparse_prefill(
q,
compressed_kv,
topk_csr,
attn_sink,
self.softmax_scale,
extend_kv=swa_kv,
extend_csr=state.swa_csr,
)
if inv_rope is not None:
_apply_inverse_rope(out, inv_rope)
return out
def _opus_compressed_inputs(
self,
state: _OpusPrefillState,
layer_id: int,
compress_ratio: int,
core_attn_metadata: DSV4AttnMetadata,
) -> Tuple[torch.Tensor, Csr]:
"""The bf16 compressed history of the layer's kv_source and the CSR of the
ratio's current top-k into it."""
pool = self.token_to_kv_pool
cache = state.cache
c_cache = pool.get_extra_key_buffer(layer_id)
entry = state.compressed_kv.get(compress_ratio)
if entry is None or entry[0] != c_cache.data_ptr():
c_page_size = pool.get_extra_key_page_size(layer_id)
gather = cache.ensure_compressed(
compress_ratio, core_attn_metadata.page_table, c_page_size
)
num_rows = gather.flat_token_ids.shape[0]
rows = self.opus_compressed_workspaces[compress_ratio].get(num_rows)
dequantize_k_cache_paged(
c_cache,
gather.flat_token_ids,
page_size=c_page_size,
out=rows,
layout=pool.get_extra_key_layout(layer_id),
)
entry = (c_cache.data_ptr(), rows.view(num_rows, -1))
state.compressed_kv[compress_ratio] = entry
compressed_kv = entry[1]
topk_csr = state.topk_csr.get(compress_ratio)
if topk_csr is None:
raw_indices = core_attn_metadata.sparse_raw_indices(compress_ratio)
indices, lens = cache.combine_compressed(
compress_ratio, raw_indices[: cache.num_qo_tokens]
)
topk_csr = combined_to_csr(indices, lens, 0, compressed_kv.shape[0])
state.topk_csr[compress_ratio] = topk_csr
return compressed_kv, topk_csr
def _drop_opus_topk_csr(self, compress_ratio: int) -> None:
if self._opus_prefill_state is not None:
self._opus_prefill_state.topk_csr.pop(compress_ratio, None)
def _forward_attention(
self,
q: torch.Tensor,
@@ -2875,6 +3093,16 @@ class DeepseekV4HipRadixBackend(
if isinstance(core_attn_metadata, DSV4AttnMetadata):
if save_kv_cache:
self.store_cache(layer_id, swa_k, forward_batch)
if self._opus_prefill_applies(q, forward_batch, compress_ratio):
return self._forward_prefill_opus(
q,
layer_id,
compress_ratio,
forward_batch,
core_attn_metadata,
attn_sink,
inv_rope,
)
swa_k_cache = token_to_kv_pool.get_swa_key_buffer_radix(layer_id)
extra_k_cache, extra_indices, extra_topk_lengths = None, None, None
@@ -3029,6 +3257,7 @@ class DeepseekV4HipRadixBackend(
self.forward_metadata.core_metadata.drop_folded_sparse_indices(
layer.compress_ratio
)
self._drop_opus_topk_csr(layer.compress_ratio)
if (
forward_batch.forward_mode.is_decode()
or forward_batch.forward_mode.is_target_verify()