[AMD][DCP 2/N] Enable aiter asm ps for kimi k3 prefill (#34438)

This commit is contained in:
billishyahao
2026-10-03 23:34:45 -07:00
committed by GitHub
parent 048b2a07f3
commit 6cc661f223
@@ -134,6 +134,23 @@ class WrapperDispatch(Enum):
CROSS_ATTENTION = auto()
@dataclass
class MlaPrefillPsMetadata:
qo_indptr: torch.Tensor
kv_indptr: torch.Tensor
kv_indices: torch.Tensor
work_metadata: torch.Tensor
work_indptr: torch.Tensor
work_info_set: torch.Tensor
reduce_indptr: torch.Tensor
reduce_final_map: torch.Tensor
reduce_partial_map: torch.Tensor
max_q_len: int
is_causal: bool
need_lse: bool
num_partial_tiles: int
@dataclass
class ForwardMetadata:
kv_indptr: torch.Tensor
@@ -153,7 +170,6 @@ class ForwardMetadata:
custom_mask: Optional[torch.Tensor] = None
mask_indptr: Optional[torch.Tensor] = None
max_extend_len: Optional[int] = None
fp8_prefill_kv_indices: Optional[torch.Tensor] = None
swa_page_table: Optional[torch.Tensor] = None
# full->SWA translated out_cache_loc (SWA KV-store write target)
swa_out_cache_loc: Optional[torch.Tensor] = None
@@ -166,6 +182,9 @@ class ForwardMetadata:
asm_ctx_cu_k: Optional[torch.Tensor] = None
# (page_indptr, page_ids, last_page_len); None means use the token-level table
paged_kv_view: Optional[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = None
prefill_ps_metadata: Optional[MlaPrefillPsMetadata] = None
chunked_skip_prefix_ps_metadata: Optional[MlaPrefillPsMetadata] = None
chunked_prefix_ps_metadatas: Optional[list[Optional[MlaPrefillPsMetadata]]] = None
def _build_paged_kv_view(
@@ -206,6 +225,10 @@ def _paged_prefill_asm_supports_gqa(num_q_heads: int, num_kv_heads: int) -> bool
_AITER_PARTITION_SIZE_ROCM = 256
# Query rows one asm-prefill work tile covers. The scheduler, the partial
# buffers and mla_reduce_v1 must all agree on it.
_PREFILL_TILE_Q = 256
_DCP_VERIFY_TABLE_COLS_PER_BLOCK = 128
@@ -408,6 +431,7 @@ class AiterAttnBackend(AttentionBackend):
(max_bs + 1,), dtype=torch.int64, device=model_runner.device
)
self._kv_indices_scratch: Optional[torch.Tensor] = None
self._arange_buf: Optional[torch.Tensor] = None
# Create prefill indices updater
if not skip_prefill:
@@ -779,6 +803,14 @@ class AiterAttnBackend(AttentionBackend):
dtype_kv=dtype,
)
@property
def _prefill_qlen_granularity(self) -> int:
"""Query rows per scheduling unit: one tile's worth, divided across the
query heads the kernel processes together."""
return _PREFILL_TILE_Q // (
self.fp8_prefill_num_head // self.fp8_prefill_num_kv_head
)
def make_mla_prefill_ps_meta_data_buffer(
self, batch_size: int, max_qlen: int, qlen_granularity: int
):
@@ -835,12 +867,12 @@ class AiterAttnBackend(AttentionBackend):
reduce_final_map: torch.Tensor,
reduce_partial_map: torch.Tensor,
is_causal: bool = True,
need_lse: bool = False,
):
gqa_ratio = self.fp8_prefill_num_head // self.fp8_prefill_num_kv_head
num_heads_k = self.fp8_prefill_num_kv_head
tile_q = 256
qhead_granularity = gqa_ratio
qlen_granularity = tile_q // qhead_granularity
qlen_granularity = self._prefill_qlen_granularity
kvlen_granularity = 128
block_size = 1
@@ -865,6 +897,7 @@ class AiterAttnBackend(AttentionBackend):
kvlen_granularity=kvlen_granularity,
block_size=block_size,
is_causal=is_causal,
need_lse=need_lse,
)
# for page size > 1 useful conversion function
@@ -1158,6 +1191,16 @@ class AiterAttnBackend(AttentionBackend):
)
return self._kv_indices_scratch[:required_tokens]
def _get_arange(self, length: int) -> torch.Tensor:
"""arange(length) grown to the next power of two."""
if self._arange_buf is None or self._arange_buf.numel() < length:
self._arange_buf = torch.arange(
1 << max(length - 1, 0).bit_length(),
device=self.device,
dtype=torch.int32,
)
return self._arange_buf[:length]
def _asm_context_prefill_indices(
self, forward_batch: ForwardBatch, bs: int, num_kv_slots: int
):
@@ -1508,6 +1551,20 @@ class AiterAttnBackend(AttentionBackend):
v: torch.Tensor,
layer: RadixAttention,
):
out, _ = self._mla_fp8_prefill_attn_ps(
q, k, v, layer, self.forward_metadata.prefill_ps_metadata
)
return out
def _mla_fp8_prefill_attn_ps(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
layer: RadixAttention,
ps: MlaPrefillPsMetadata,
):
"""Run the asm prefill over one PS metadata, returning (out, lse)."""
total_q = q.shape[0]
nhead = layer.tp_q_head_num
v_head_dim = layer.v_head_dim
@@ -1529,25 +1586,21 @@ class AiterAttnBackend(AttentionBackend):
v = v.to(fp8_dtype)
one_scale = torch.ones((), dtype=torch.float32, device=q.device)
tile_q = 256
reduce_indptr = self.forward_metadata.reduce_indptr
reduce_final_map = self.forward_metadata.reduce_final_map
reduce_partial_map = self.forward_metadata.reduce_partial_map
partial_q = max(ps.num_partial_tiles, 1) * _PREFILL_TILE_Q
logits = torch.empty(
(reduce_partial_map.size(0) * tile_q, nhead, v_head_dim),
(partial_q, nhead, v_head_dim),
dtype=torch.float32,
device=q.device,
)
attn_lse = torch.empty(
(reduce_partial_map.size(0) * tile_q, nhead),
(partial_q, nhead),
dtype=torch.float32,
device=q.device,
)
final_lse = torch.empty(
(total_q, nhead),
dtype=torch.float32,
device=q.device,
final_lse = (
torch.empty((total_q, nhead), dtype=torch.float32, device=q.device)
if ps.need_lse
else None
)
output = q.new_empty(
(total_q, nhead, v_head_dim),
@@ -1558,14 +1611,14 @@ class AiterAttnBackend(AttentionBackend):
q,
k,
v,
self.forward_metadata.qo_indptr,
self.forward_metadata.kv_indptr,
self.forward_metadata.fp8_prefill_kv_indices,
self.forward_metadata.work_indptr,
self.forward_metadata.work_info_set,
self.forward_metadata.max_q_len,
ps.qo_indptr,
ps.kv_indptr,
ps.kv_indices,
ps.work_indptr,
ps.work_info_set,
ps.max_q_len,
layer.scaling,
True,
ps.is_causal,
logits,
attn_lse,
output,
@@ -1576,16 +1629,23 @@ class AiterAttnBackend(AttentionBackend):
mla_reduce_v1(
logits,
attn_lse,
reduce_indptr,
reduce_final_map,
reduce_partial_map,
tile_q,
ps.reduce_indptr,
ps.reduce_final_map,
ps.reduce_partial_map,
_PREFILL_TILE_Q,
# Prefill PS metadata has no split cap; 0 keeps AITER's default reduce sizing.
0,
output,
final_lse,
)
return output[:, : layer.tp_q_head_num, :] if head_pad else output
if head_pad:
output = output[:, : layer.tp_q_head_num, :]
if final_lse is not None:
# Slicing the padded head axis leaves the kernel's stride
# behind, and merge_state reads its inputs as contiguous.
output = output.contiguous()
final_lse = final_lse[:, : layer.tp_q_head_num].contiguous()
return output, final_lse
def _kv_index_blocks(self, bs: int) -> int:
if self.max_context_len < _KV_INDEX_BLOCKS_MIN_CONTEXT:
@@ -2092,48 +2152,21 @@ class AiterAttnBackend(AttentionBackend):
qo_indptr = self.mla_indices_updater_prefill.qo_indptr
kv_indptr = self.mla_indices_updater_prefill.kv_indptr
work_metadata = None
work_indptr = None
work_info_set = None
reduce_indptr = None
reduce_final_map = None
reduce_partial_map = None
fp8_prefill_kv_indices = None
# fp8 PS-ASM prefill memory-faults on gfx950 for 12-head
# (zero-pad) models; keep it off and use flash-attn fallback.
if self.use_fp8_prefill_attn and self.head_pad_mode != "zero":
tile_q = 256
qlen_granularity = tile_q // (
self.fp8_prefill_num_head // self.fp8_prefill_num_kv_head
)
(
work_metadata,
work_indptr,
work_info_set,
reduce_indptr,
reduce_final_map,
reduce_partial_map,
) = self.make_mla_prefill_ps_meta_data_buffer(
bs, max_q_len, qlen_granularity
)
self.make_mla_prefill_ps_meta_data(
qo_indptr,
kv_indptr,
forward_batch.seq_lens,
work_metadata,
work_indptr,
work_info_set,
reduce_indptr,
reduce_final_map,
reduce_partial_map,
prefill_ps_metadata = None
if self.use_fp8_prefill_attn:
prefill_ps_metadata = self._build_prefill_ps_metadata(
qo_indptr=qo_indptr,
kv_indptr=kv_indptr,
kv_lens_cpu=forward_batch.seq_lens_cpu,
num_kv_tokens=forward_batch.seq_lens_sum,
max_q_len=max_q_len,
# The keys are the whole sequence, and the extend
# tokens sit at its tail.
is_causal=True,
)
total_s = forward_batch.seq_lens_sum
fp8_prefill_kv_indices = torch.arange(
total_s, device=self.device, dtype=torch.int32
need_lse=False,
# Keep the static bound this path has always sized its
# partial buffers with, rather than the emitted count.
exact_partial_count=False,
)
self.forward_metadata = ForwardMetadata(
@@ -2143,13 +2176,7 @@ class AiterAttnBackend(AttentionBackend):
self.kv_last_page_len[:bs],
max_q_len,
self.mla_indices_updater_prefill.max_kv_len,
work_metadata=work_metadata,
work_info_set=work_info_set,
work_indptr=work_indptr,
reduce_indptr=reduce_indptr,
reduce_final_map=reduce_final_map,
reduce_partial_map=reduce_partial_map,
fp8_prefill_kv_indices=fp8_prefill_kv_indices,
prefill_ps_metadata=prefill_ps_metadata,
)
else:
self.indices_updater_prefill.update(
@@ -2935,7 +2962,143 @@ class AiterAttnBackend(AttentionBackend):
def init_mha_chunk_metadata(
self, forward_batch: ForwardBatch, disable_flashinfer_ragged: bool = False
) -> None:
pass
"""Build the chunked-prefix route's PS metadata, once per forward.
Its attentions each attend a subset of the sequence, while
init_forward_metadata's PS metadata covers the whole sequence.
"""
# The one-shot core calls this hook too, with num_prefix_chunks 0.
if not forward_batch.num_prefix_chunks or not self.use_fp8_prefill_attn:
return
# Both passes query the extend tokens, so they share qo_indptr.
extend_lens_cpu = torch.tensor(
forward_batch.extend_seq_lens_cpu, dtype=torch.int32
)
qo_indptr_cpu = torch.zeros(len(extend_lens_cpu) + 1, dtype=torch.int32)
qo_indptr_cpu[1:] = torch.cumsum(extend_lens_cpu, dim=0)
qo_indptr = self.forward_metadata.qo_indptr
max_q_len = self.forward_metadata.max_q_len
# disable_flashinfer_ragged asks for no ragged pass. The skip-prefix
# pass is this backend's ragged one, so leave it on the varlen path.
if not disable_flashinfer_ragged:
self.forward_metadata.chunked_skip_prefix_ps_metadata = (
self._build_prefill_ps_metadata(
qo_indptr=qo_indptr,
qo_indptr_cpu=qo_indptr_cpu,
kv_indptr=qo_indptr,
kv_indptr_cpu=qo_indptr_cpu,
kv_lens_cpu=extend_lens_cpu,
num_kv_tokens=int(qo_indptr_cpu[-1]),
max_q_len=max_q_len,
is_causal=True,
need_lse=True,
)
)
metadatas: list[Optional[MlaPrefillPsMetadata]] = []
for i in range(forward_batch.num_prefix_chunks):
chunk_lens_cpu = forward_batch.prefix_chunk_seq_lens_cpu[i].to(torch.int32)
# An empty kv range has no PS metadata shape.
# _forward_extend_prefix_chunk falls back to varlen there.
if forward_batch.prefix_chunk_has_zero_kv[i]:
metadatas.append(None)
continue
chunk_indptr_cpu = torch.zeros(len(chunk_lens_cpu) + 1, dtype=torch.int32)
chunk_indptr_cpu[1:] = torch.cumsum(chunk_lens_cpu, dim=0)
metadatas.append(
self._build_prefill_ps_metadata(
qo_indptr=qo_indptr,
qo_indptr_cpu=qo_indptr_cpu,
kv_indptr=forward_batch.prefix_chunk_cu_seq_lens[i],
kv_indptr_cpu=chunk_indptr_cpu,
kv_lens_cpu=chunk_lens_cpu,
num_kv_tokens=forward_batch.prefix_chunk_num_tokens[i],
max_q_len=max_q_len,
# Every key in a prefix chunk precedes every extend token.
is_causal=False,
need_lse=True,
)
)
self.forward_metadata.chunked_prefix_ps_metadatas = metadatas
def _build_prefill_ps_metadata(
self,
*,
qo_indptr: torch.Tensor,
kv_indptr: torch.Tensor,
kv_lens_cpu: torch.Tensor,
num_kv_tokens: int,
max_q_len: int,
is_causal: bool,
need_lse: bool,
qo_indptr_cpu: Optional[torch.Tensor] = None,
kv_indptr_cpu: Optional[torch.Tensor] = None,
exact_partial_count: bool = True,
) -> MlaPrefillPsMetadata:
"""Plan one asm-prefill PS metadata over the key set the caller names."""
(
work_metadata,
work_indptr,
work_info_set,
reduce_indptr,
reduce_final_map,
reduce_partial_map,
) = self.make_mla_prefill_ps_meta_data_buffer(
len(kv_lens_cpu), max_q_len, self._prefill_qlen_granularity
)
self.make_mla_prefill_ps_meta_data(
qo_indptr if qo_indptr_cpu is None else qo_indptr_cpu,
kv_indptr if kv_indptr_cpu is None else kv_indptr_cpu,
kv_lens_cpu,
work_metadata,
work_indptr,
work_info_set,
reduce_indptr,
reduce_final_map,
reduce_partial_map,
is_causal=is_causal,
need_lse=need_lse,
)
if exact_partial_count:
# One D2H sync per PS metadata, once per forward rather than per
# layer. get_ps_metadata_v1 already pays several.
num_partial_tiles = int(reduce_indptr[-1].item())
if need_lse:
assert num_partial_tiles > 0, (
"the scheduler emitted no partial tiles, so mla_reduce_v1 "
"would write no LSE for this partition"
)
if num_partial_tiles:
max_partial_row = int(reduce_partial_map[:num_partial_tiles].max())
assert (
max_partial_row + _PREFILL_TILE_Q
<= num_partial_tiles * _PREFILL_TILE_Q
), (
f"reduce_partial_map reaches row {max_partial_row} and the "
f"tile is {_PREFILL_TILE_Q} rows, but the partial buffers "
f"hold only {num_partial_tiles * _PREFILL_TILE_Q} rows"
)
else:
num_partial_tiles = reduce_partial_map.size(0)
return MlaPrefillPsMetadata(
qo_indptr=qo_indptr,
kv_indptr=kv_indptr,
# The k/v handed to the kernel are contiguous and in key order, so
# the page table is the identity.
kv_indices=self._get_arange(num_kv_tokens),
work_metadata=work_metadata,
work_indptr=work_indptr,
work_info_set=work_info_set,
reduce_indptr=reduce_indptr,
reduce_final_map=reduce_final_map,
reduce_partial_map=reduce_partial_map,
max_q_len=max_q_len,
is_causal=is_causal,
need_lse=need_lse,
num_partial_tiles=num_partial_tiles,
)
def _forward_extend_prefix_chunk(
self,
@@ -2946,6 +3109,10 @@ class AiterAttnBackend(AttentionBackend):
forward_batch: ForwardBatch,
):
idx = forward_batch.prefix_chunk_idx
ps_metadatas = self.forward_metadata.chunked_prefix_ps_metadatas
if ps_metadatas is not None and ps_metadatas[idx] is not None:
return self._mla_fp8_prefill_attn_ps(q, k, v, layer, ps_metadatas[idx])
output, lse = flash_attn_varlen_func(
q,
k,
@@ -2958,6 +3125,8 @@ class AiterAttnBackend(AttentionBackend):
causal=False,
return_lse=True,
)[:2]
# the merge_state_triton needs lse layout [tokens, heads].
# See https://github.com/sgl-project/sglang/blob/98fce73d5bd0a25afe7d68443d314190b1c47e64/python/sglang/kernels/ops/attention/merge_state.py#L13-L15
return output, lse.transpose(0, 1).contiguous()
def _forward_extend_skip_prefix(
@@ -2967,6 +3136,10 @@ class AiterAttnBackend(AttentionBackend):
v: torch.Tensor,
layer: RadixAttention,
):
ps = self.forward_metadata.chunked_skip_prefix_ps_metadata
if ps is not None:
return self._mla_fp8_prefill_attn_ps(q, k, v, layer, ps)
qo_indptr = self.forward_metadata.qo_indptr
max_q_len = self.forward_metadata.max_q_len
output, lse = flash_attn_varlen_func(
@@ -2981,6 +3154,8 @@ class AiterAttnBackend(AttentionBackend):
causal=True,
return_lse=True,
)[:2]
# the merge_state_triton needs lse layout [tokens, heads].
# See https://github.com/sgl-project/sglang/blob/98fce73d5bd0a25afe7d68443d314190b1c47e64/python/sglang/kernels/ops/attention/merge_state.py#L13-L15
return output, lse.transpose(0, 1).contiguous()
@staticmethod
@@ -3199,7 +3374,7 @@ class AiterAttnBackend(AttentionBackend):
if forward_batch.mha_return_lse:
return self._forward_extend_skip_prefix(q, k, v, layer)
if self.dcp_world_size > 1:
if self.use_fp8_prefill_attn and self.head_pad_mode != "zero":
if self.use_fp8_prefill_attn:
return self.mla_fp8_prefill_attn(q, k, v, layer)
return flash_attn_varlen_func(
q,
@@ -3213,7 +3388,7 @@ class AiterAttnBackend(AttentionBackend):
causal=True,
)
if kv_indices.shape[0] == 0 or extend_no_prefix:
if self.use_fp8_prefill_attn and self.head_pad_mode != "zero":
if self.use_fp8_prefill_attn:
output = self.mla_fp8_prefill_attn(
q,
k,
@@ -3247,7 +3422,6 @@ class AiterAttnBackend(AttentionBackend):
if (
self.use_fp8_prefill_attn
and self.head_pad_mode != "zero"
and layer.kv_b_proj.weight.dtype == torch.uint8
):
# MXFP4 weights + FP8 prefill: fuse GEMM, nope/v split, and k_pe cat
@@ -3287,7 +3461,7 @@ class AiterAttnBackend(AttentionBackend):
== forward_batch.extend_seq_lens.shape
)
if self.use_fp8_prefill_attn and self.head_pad_mode != "zero":
if self.use_fp8_prefill_attn:
return self.mla_fp8_prefill_attn(q, k, v, layer)
else:
return flash_attn_varlen_func(