[AMD][V4.1][*/N] OPUS sparse prefill on gfx950 through layout conversion (#42017)
Co-authored-by: HAI <hixiao@gmail.com> Co-authored-by: thomawan <thomawan@amd.com>
This commit is contained in:
co-authored by
HAI
thomawan
parent
f45c004e18
commit
3c4e218779
@@ -0,0 +1,101 @@
|
||||
"""DeepSeek-V4 sparse prefill attention on gfx950 through aiter's OPUS kernel.
|
||||
|
||||
OPUS attends over two bf16 row sources, each addressed by a CSR index list. The
|
||||
caller dequantizes rows into bf16 workspaces and builds padded per-query lists
|
||||
(``SparsePrefillChunkCache``); this module turns those lists into CSR and runs
|
||||
the kernel.
|
||||
"""
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
# (int32 indices, int32 indptr [T + 1])
|
||||
Csr = Tuple[torch.Tensor, torch.Tensor]
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _combined_to_csr_kernel(
|
||||
indices_ptr,
|
||||
indices_stride,
|
||||
lens_ptr,
|
||||
indptr_ptr,
|
||||
out_ptr,
|
||||
row_start,
|
||||
row_end,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
# OPUS reads every CSR entry, so -1 holes inside the scanned prefix and
|
||||
# entries outside [row_start, row_end) are dropped here.
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
cols = tl.arange(0, BLOCK)
|
||||
length = tl.load(lens_ptr + row)
|
||||
idx = tl.load(
|
||||
indices_ptr + row * indices_stride + cols, mask=cols < length, other=-1
|
||||
)
|
||||
keep = (idx >= row_start) & (idx < row_end)
|
||||
dst = tl.cumsum(keep.to(tl.int32), axis=0) - 1
|
||||
start = tl.load(indptr_ptr + row)
|
||||
tl.store(out_ptr + start + dst, idx - row_start, mask=keep)
|
||||
|
||||
|
||||
def combined_to_csr(
|
||||
indices: torch.Tensor, lens: torch.Tensor, row_start: int, row_end: int
|
||||
) -> Csr:
|
||||
"""int32 [T, W] padded index lists with per-row scanned lengths [T] -> the CSR
|
||||
(T * W capacity) of their entries in [row_start, row_end), rebased to row_start."""
|
||||
num_rows, width = indices.shape
|
||||
device = indices.device
|
||||
cols = torch.arange(width, device=device, dtype=torch.int32)
|
||||
keep = (
|
||||
(cols[None, :] < lens[:, None]) & (indices >= row_start) & (indices < row_end)
|
||||
)
|
||||
indptr = torch.zeros(num_rows + 1, dtype=torch.int32, device=device)
|
||||
torch.cumsum(keep.sum(dim=1, dtype=torch.int32), dim=0, out=indptr[1:])
|
||||
csr = torch.empty(max(num_rows * width, 1), dtype=torch.int32, device=device)
|
||||
if num_rows > 0:
|
||||
_combined_to_csr_kernel[(num_rows,)](
|
||||
indices,
|
||||
indices.stride(0),
|
||||
lens,
|
||||
indptr,
|
||||
csr,
|
||||
row_start,
|
||||
row_end,
|
||||
BLOCK=triton.next_power_of_2(width),
|
||||
)
|
||||
return csr, indptr
|
||||
|
||||
|
||||
def opus_sparse_prefill(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
csr: Csr,
|
||||
attn_sink: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
extend_kv: Optional[torch.Tensor] = None,
|
||||
extend_csr: Optional[Csr] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Attention of q [T, H, 512] bf16 over the bf16 rows of kv [R, 512] named by
|
||||
csr, then those of extend_kv named by extend_csr, with a per-head sink."""
|
||||
from aiter.ops.pa_sparse_prefill_opus import pa_sparse_prefill_opus
|
||||
|
||||
indices, indptr = csr
|
||||
if extend_kv is None:
|
||||
# every query reads an empty row of the unused extend source
|
||||
extend_kv = kv[:1]
|
||||
extend_csr = (indices[:1], torch.zeros_like(indptr))
|
||||
extend_indices, extend_indptr = extend_csr
|
||||
return pa_sparse_prefill_opus(
|
||||
q.contiguous(),
|
||||
kv,
|
||||
indices,
|
||||
indptr,
|
||||
extend_kv,
|
||||
extend_indices,
|
||||
extend_indptr,
|
||||
attn_sink,
|
||||
softmax_scale,
|
||||
)
|
||||
@@ -1579,6 +1579,9 @@ class Envs:
|
||||
SGLANG_DSV41_TORCH_PREFILL_INDEXER = EnvBool(False)
|
||||
SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False)
|
||||
SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True)
|
||||
# gfx950 DeepSeek-V4.1 prefill: attend with aiter's OPUS sparse kernel over a bf16
|
||||
# dequant of the chunk's SWA and compressed history, instead of the decode kernel.
|
||||
SGLANG_OPT_HIP_OPUS_SPARSE_PREFILL = EnvBool(False)
|
||||
# DSpark draft block on the HIP radix backend: build the attention metadata inside the
|
||||
# draft CUDA graph from the raw inputs instead of eagerly before every replay.
|
||||
SGLANG_HIP_DSPARK_DRAFT_RAW_METADATA = EnvBool(_default_hip)
|
||||
|
||||
@@ -44,10 +44,16 @@ from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import (
|
||||
from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import (
|
||||
SOFTMAX_SCALE as SWAPAB_SOFTMAX_SCALE,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.dequant_k_cache import dequantize_k_cache_paged
|
||||
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
|
||||
from sglang.kernels.ops.attention.dsv4.metadata_kernel import (
|
||||
init_compression_metadata as _init_compression_metadata_triton,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.opus_sparse_prefill_hip import (
|
||||
Csr,
|
||||
combined_to_csr,
|
||||
opus_sparse_prefill,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.swapab_gluon_hip import swapab_attention
|
||||
from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import (
|
||||
BuildCausalSwaPageIndices,
|
||||
@@ -81,12 +87,18 @@ from sglang.srt.layers.attention.dsv4.metadata import (
|
||||
copy_metadata,
|
||||
maybe_copy_inplace,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.sparse_prefill_utils import (
|
||||
SparsePrefillChunkCache,
|
||||
SparsePrefillWorkspace,
|
||||
)
|
||||
from sglang.srt.layers.attention.hip_flash_mla import (
|
||||
_apply_inverse_rope,
|
||||
flash_mla_with_kvcache_entrypoint,
|
||||
hip_attn_kv_splits,
|
||||
resolve_hip_flashmla_backend,
|
||||
)
|
||||
from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask
|
||||
from sglang.srt.layers.cp.utils import is_cp_active
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.mem_cache.dsv41_request_window import window_layout
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
@@ -97,7 +109,7 @@ from sglang.srt.runtime_context import (
|
||||
)
|
||||
from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc
|
||||
from sglang.srt.speculative.ragged_verify import resolve_ragged_verify_layout
|
||||
from sglang.srt.utils import ceil_align
|
||||
from sglang.srt.utils import ceil_align, is_gfx95_supported
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sgl_kernel.flash_mla import FlashMLASchedMeta
|
||||
@@ -130,6 +142,34 @@ def _pad_last_dim(x: T, multiples_of: int = PAGE_INDEX_ALIGNED_SIZE) -> T:
|
||||
return F.pad(x, pad=(0, target_size - curr_size), mode="constant", value=-1)
|
||||
|
||||
|
||||
# OPUS prefill applies from this many query rows; below it the decode kernel splits KV
|
||||
# and the dequant is not amortized.
|
||||
_OPUS_PREFILL_MIN_TOKENS = 1024
|
||||
# The OPUS workspace holds every request's whole compressed history, dequantized once
|
||||
# per kv_source layer. Past this many history rows per query row that dequant costs
|
||||
# more than OPUS saves (gfx950, uniformly random top-k): a ratio-2 source serves fewer
|
||||
# layers than the ratio-1 one.
|
||||
_OPUS_PREFILL_MAX_ROWS_PER_TOKEN = {1: 256, 2: 128}
|
||||
# One ratio's bf16 workspace stays within 1 GiB.
|
||||
_OPUS_PREFILL_MAX_WORKSPACE_ROWS = 1 << 20
|
||||
|
||||
|
||||
@dataclass
|
||||
class _OpusPrefillState:
|
||||
"""OPUS prefill inputs of one forward. A V4.1 ratio-1/2 layer reads the
|
||||
compressed KV of its kv_source layer and the top-k of its index_source layer,
|
||||
so each is built once and reused by the layers that follow its source."""
|
||||
|
||||
cache: SparsePrefillChunkCache
|
||||
# the SWA rows every layer lays out alike
|
||||
swa_csr: Csr
|
||||
# ratio -> (data_ptr of the compressed cache, its bf16 rows)
|
||||
compressed_kv: Dict[int, Tuple[int, torch.Tensor]] = field(default_factory=dict)
|
||||
# ratio -> CSR of the top-k into compressed_kv[ratio]; dropped when the indexer
|
||||
# rewrites that ratio's top-k
|
||||
topk_csr: Dict[int, Csr] = field(default_factory=dict)
|
||||
|
||||
|
||||
def _fold_lengths_into_index_lists(
|
||||
indices: torch.Tensor,
|
||||
lengths: Optional[torch.Tensor],
|
||||
@@ -625,11 +665,17 @@ class DSV4AttnMetadata:
|
||||
self.c0_flashmla_metadata = _create_flashmla_metadata()
|
||||
self.c4_flashmla_metadata = _create_flashmla_metadata()
|
||||
self.c128_flashmla_metadata = _create_flashmla_metadata()
|
||||
# -1 filled like the page indices: identity rows write only their reachable columns,
|
||||
# and the OPUS prefill reads the raw indices without the top-k lengths
|
||||
if 1 in self.low_ratios:
|
||||
self.c1_sparse_raw_indices = torch.empty_like(self.c1_sparse_page_indices)
|
||||
self.c1_sparse_raw_indices = torch.full_like(
|
||||
self.c1_sparse_page_indices, -1
|
||||
)
|
||||
self.c1_flashmla_metadata = _create_flashmla_metadata()
|
||||
if 2 in self.low_ratios:
|
||||
self.c2_sparse_raw_indices = torch.empty_like(self.c2_sparse_page_indices)
|
||||
self.c2_sparse_raw_indices = torch.full_like(
|
||||
self.c2_sparse_page_indices, -1
|
||||
)
|
||||
self.c2_flashmla_metadata = _create_flashmla_metadata()
|
||||
|
||||
|
||||
@@ -973,6 +1019,12 @@ class DeepseekV4HipRadixBackend(
|
||||
DSV4RawVerifyMetadata,
|
||||
DSV4RawDecodeMetadata,
|
||||
] = None
|
||||
self.opus_prefill_workspace = SparsePrefillWorkspace(self.device)
|
||||
self.opus_compressed_workspaces = {
|
||||
ratio: SparsePrefillWorkspace(self.device) for ratio in (1, 2)
|
||||
}
|
||||
# Reset with the metadata: its inputs belong to one prefill forward.
|
||||
self._opus_prefill_state: Optional[_OpusPrefillState] = None
|
||||
|
||||
def _move_to_device(self, x: List[int]) -> torch.Tensor:
|
||||
pin_tensor = torch.tensor(x, dtype=torch.int32, pin_memory=True)
|
||||
@@ -1996,6 +2048,7 @@ class DeepseekV4HipRadixBackend(
|
||||
if self.mtp_enabled and forward_batch.forward_mode.is_idle():
|
||||
return
|
||||
|
||||
self._opus_prefill_state = None
|
||||
self.forward_metadata = self._build_forward_metadata(forward_batch)
|
||||
self.init_forward_metadata_in_graph(forward_batch)
|
||||
self._refresh_fp4_prefill_workspace(forward_batch)
|
||||
@@ -2210,6 +2263,7 @@ class DeepseekV4HipRadixBackend(
|
||||
def init_forward_metadata_for_breakable_cuda_graph_capture(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> DSV4Metadata:
|
||||
self._opus_prefill_state = None
|
||||
if not self.low_ratios:
|
||||
self.forward_metadata = self._build_forward_metadata(
|
||||
forward_batch,
|
||||
@@ -2253,6 +2307,7 @@ class DeepseekV4HipRadixBackend(
|
||||
*,
|
||||
static_forward_batch: Optional[ForwardBatch] = None,
|
||||
) -> None:
|
||||
self._opus_prefill_state = None
|
||||
if not self.low_ratios:
|
||||
replay_batch = (
|
||||
static_forward_batch
|
||||
@@ -2821,6 +2876,169 @@ class DeepseekV4HipRadixBackend(
|
||||
cache_k=swa_k,
|
||||
)
|
||||
|
||||
def _opus_prefill_applies(
|
||||
self, q: torch.Tensor, forward_batch: ForwardBatch, compress_ratio: int
|
||||
) -> bool:
|
||||
if not envs.SGLANG_OPT_HIP_OPUS_SPARSE_PREFILL.get():
|
||||
return False
|
||||
if (
|
||||
compress_ratio not in (0, 1, 2)
|
||||
or self.is_draft_worker
|
||||
or not forward_batch.forward_mode.is_extend_without_speculative()
|
||||
or q.ndim != 3
|
||||
or q.shape[0] < _OPUS_PREFILL_MIN_TOKENS
|
||||
or q.shape[-1] != 512
|
||||
or q.dtype != torch.bfloat16
|
||||
or not is_gfx95_supported()
|
||||
or self.token_to_kv_pool.request_window is not None
|
||||
or self.forward_metadata.late_layer_tail is not None
|
||||
or is_cp_active(forward_batch)
|
||||
or forward_batch.seq_lens_cpu is None
|
||||
or forward_batch.extend_seq_lens_cpu is None
|
||||
):
|
||||
return False
|
||||
if compress_ratio == 0:
|
||||
return True
|
||||
seq_lens = forward_batch.seq_lens_cpu
|
||||
history_rows = len(seq_lens) * max(int(seq_lens.max()) // compress_ratio, 1)
|
||||
return history_rows <= min(
|
||||
_OPUS_PREFILL_MAX_ROWS_PER_TOKEN[compress_ratio] * q.shape[0],
|
||||
_OPUS_PREFILL_MAX_WORKSPACE_ROWS,
|
||||
)
|
||||
|
||||
def _build_opus_prefill_cache(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
core_attn_metadata: DSV4AttnMetadata,
|
||||
num_tokens: int,
|
||||
) -> SparsePrefillChunkCache:
|
||||
seq_lens_cpu = forward_batch.seq_lens_cpu.tolist()
|
||||
total_swa = sum(
|
||||
min(int(seq_len), int(extend_len) + SWA_WINDOW - 1)
|
||||
for seq_len, extend_len in zip(
|
||||
seq_lens_cpu, forward_batch.extend_seq_lens_cpu, strict=True
|
||||
)
|
||||
)
|
||||
extend_seq_lens = forward_batch.extend_seq_lens.to(torch.int32)
|
||||
query_pos = core_attn_metadata.seq_lens_casual[:num_tokens] - 1
|
||||
assert query_pos.shape[0] == num_tokens, (query_pos.shape, num_tokens)
|
||||
return SparsePrefillChunkCache.build(
|
||||
seq_lens=forward_batch.seq_lens.to(torch.int32),
|
||||
extend_seq_lens=extend_seq_lens,
|
||||
query_lens=extend_seq_lens,
|
||||
query_pos=query_pos.to(torch.int32),
|
||||
req_pool_indices=forward_batch.req_pool_indices.to(torch.int32),
|
||||
req_to_token=self.req_to_token,
|
||||
full_to_swa=self.token_to_kv_pool.full_to_swa_index_mapping,
|
||||
swa_window_size=SWA_WINDOW,
|
||||
swa_page_size=self.token_to_kv_pool.swa_kv_pool.page_size,
|
||||
num_qo_tokens=num_tokens,
|
||||
max_seq_len=max(seq_lens_cpu),
|
||||
total_swa=total_swa,
|
||||
)
|
||||
|
||||
def _forward_prefill_opus(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
layer_id: int,
|
||||
compress_ratio: int,
|
||||
forward_batch: ForwardBatch,
|
||||
core_attn_metadata: DSV4AttnMetadata,
|
||||
attn_sink: torch.Tensor,
|
||||
inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]],
|
||||
) -> torch.Tensor:
|
||||
"""Sparse prefill through OPUS over the layer's SWA rows and, for ratios 1 / 2,
|
||||
the compressed history its top-k selects."""
|
||||
pool = self.token_to_kv_pool
|
||||
num_tokens = q.shape[0]
|
||||
state = self._opus_prefill_state
|
||||
if state is None or state.cache.num_qo_tokens != num_tokens:
|
||||
cache = self._build_opus_prefill_cache(
|
||||
forward_batch, core_attn_metadata, num_tokens
|
||||
)
|
||||
swa_csr = combined_to_csr(
|
||||
cache.c0_combined_indices,
|
||||
cache.c0_combined_lens,
|
||||
0,
|
||||
cache.swa_token_ids.shape[0],
|
||||
)
|
||||
state = self._opus_prefill_state = _OpusPrefillState(cache, swa_csr)
|
||||
cache = state.cache
|
||||
num_swa_rows = cache.swa_token_ids.shape[0]
|
||||
swa_rows = self.opus_prefill_workspace.get(num_swa_rows)
|
||||
dequantize_k_cache_paged(
|
||||
pool.get_swa_key_buffer_radix(layer_id),
|
||||
cache.swa_token_ids,
|
||||
page_size=cache.swa_page_size,
|
||||
out=swa_rows,
|
||||
layout=pool.get_swa_key_layout(),
|
||||
)
|
||||
swa_kv = swa_rows.view(num_swa_rows, -1)
|
||||
if compress_ratio == 0:
|
||||
out = opus_sparse_prefill(
|
||||
q, swa_kv, state.swa_csr, attn_sink, self.softmax_scale
|
||||
)
|
||||
else:
|
||||
compressed_kv, topk_csr = self._opus_compressed_inputs(
|
||||
state, layer_id, compress_ratio, core_attn_metadata
|
||||
)
|
||||
out = opus_sparse_prefill(
|
||||
q,
|
||||
compressed_kv,
|
||||
topk_csr,
|
||||
attn_sink,
|
||||
self.softmax_scale,
|
||||
extend_kv=swa_kv,
|
||||
extend_csr=state.swa_csr,
|
||||
)
|
||||
if inv_rope is not None:
|
||||
_apply_inverse_rope(out, inv_rope)
|
||||
return out
|
||||
|
||||
def _opus_compressed_inputs(
|
||||
self,
|
||||
state: _OpusPrefillState,
|
||||
layer_id: int,
|
||||
compress_ratio: int,
|
||||
core_attn_metadata: DSV4AttnMetadata,
|
||||
) -> Tuple[torch.Tensor, Csr]:
|
||||
"""The bf16 compressed history of the layer's kv_source and the CSR of the
|
||||
ratio's current top-k into it."""
|
||||
pool = self.token_to_kv_pool
|
||||
cache = state.cache
|
||||
c_cache = pool.get_extra_key_buffer(layer_id)
|
||||
entry = state.compressed_kv.get(compress_ratio)
|
||||
if entry is None or entry[0] != c_cache.data_ptr():
|
||||
c_page_size = pool.get_extra_key_page_size(layer_id)
|
||||
gather = cache.ensure_compressed(
|
||||
compress_ratio, core_attn_metadata.page_table, c_page_size
|
||||
)
|
||||
num_rows = gather.flat_token_ids.shape[0]
|
||||
rows = self.opus_compressed_workspaces[compress_ratio].get(num_rows)
|
||||
dequantize_k_cache_paged(
|
||||
c_cache,
|
||||
gather.flat_token_ids,
|
||||
page_size=c_page_size,
|
||||
out=rows,
|
||||
layout=pool.get_extra_key_layout(layer_id),
|
||||
)
|
||||
entry = (c_cache.data_ptr(), rows.view(num_rows, -1))
|
||||
state.compressed_kv[compress_ratio] = entry
|
||||
compressed_kv = entry[1]
|
||||
topk_csr = state.topk_csr.get(compress_ratio)
|
||||
if topk_csr is None:
|
||||
raw_indices = core_attn_metadata.sparse_raw_indices(compress_ratio)
|
||||
indices, lens = cache.combine_compressed(
|
||||
compress_ratio, raw_indices[: cache.num_qo_tokens]
|
||||
)
|
||||
topk_csr = combined_to_csr(indices, lens, 0, compressed_kv.shape[0])
|
||||
state.topk_csr[compress_ratio] = topk_csr
|
||||
return compressed_kv, topk_csr
|
||||
|
||||
def _drop_opus_topk_csr(self, compress_ratio: int) -> None:
|
||||
if self._opus_prefill_state is not None:
|
||||
self._opus_prefill_state.topk_csr.pop(compress_ratio, None)
|
||||
|
||||
def _forward_attention(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
@@ -2875,6 +3093,16 @@ class DeepseekV4HipRadixBackend(
|
||||
if isinstance(core_attn_metadata, DSV4AttnMetadata):
|
||||
if save_kv_cache:
|
||||
self.store_cache(layer_id, swa_k, forward_batch)
|
||||
if self._opus_prefill_applies(q, forward_batch, compress_ratio):
|
||||
return self._forward_prefill_opus(
|
||||
q,
|
||||
layer_id,
|
||||
compress_ratio,
|
||||
forward_batch,
|
||||
core_attn_metadata,
|
||||
attn_sink,
|
||||
inv_rope,
|
||||
)
|
||||
swa_k_cache = token_to_kv_pool.get_swa_key_buffer_radix(layer_id)
|
||||
|
||||
extra_k_cache, extra_indices, extra_topk_lengths = None, None, None
|
||||
@@ -3029,6 +3257,7 @@ class DeepseekV4HipRadixBackend(
|
||||
self.forward_metadata.core_metadata.drop_folded_sparse_indices(
|
||||
layer.compress_ratio
|
||||
)
|
||||
self._drop_opus_topk_csr(layer.compress_ratio)
|
||||
if (
|
||||
forward_batch.forward_mode.is_decode()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
|
||||
Reference in New Issue
Block a user