[AMD] Reuse KV gather indices across ASM context prefill layers (#39901)

Co-authored-by: Zijie Chen <300606707+zijiecode@users.noreply.github.com>
Co-authored-by: jacky.cheng <yichiche@amd.com>
This commit is contained in:
zijie
2026-09-22 12:08:05 -07:00
committed by GitHub
co-authored by Zijie Chen jacky.cheng
parent 5c18780c31
commit 720617bb5b
2 changed files with 220 additions and 9 deletions
@@ -7,6 +7,7 @@ end to end attention solution with aiter kernels
"""
import logging
import os
from dataclasses import dataclass
from enum import Enum, auto
from typing import TYPE_CHECKING, Optional
@@ -158,6 +159,11 @@ class ForwardMetadata:
swa_out_cache_loc: Optional[torch.Tensor] = None
local_kv_lens: Optional[torch.Tensor] = None
verify_token_table: Optional[torch.Tensor] = None
# ASM context-chunk prefill: KV slots to gather and cu_seqlens_k, computed
# once per batch by AiterAttnBackend._asm_context_prefill_indices.
asm_ctx_ready: bool = False
asm_ctx_tok_idx: Optional[torch.Tensor] = None
asm_ctx_cu_k: Optional[torch.Tensor] = None
_AITER_PARTITION_SIZE_ROCM = 256
@@ -178,6 +184,12 @@ def _aiter_fp8_asm_supports_gqa(num_q_heads: int, num_kv_heads: int) -> bool:
return (num_q_heads // num_kv_heads) in _AITER_FP8_ASM_GQA_RATIOS
# Cross-check the per-batch fast indices against the generic gather (syncs).
_GFX_ASM_CTX_GATHER_CHECK = (
os.environ.get("SGLANG_GFX_ASM_CTX_GATHER_CHECK", "0") == "1"
)
def _asm_context_prefill_gather_indices(
kv_indptr: torch.Tensor,
kv_indices: torch.Tensor,
@@ -1059,6 +1071,70 @@ class AiterAttnBackend(AttentionBackend):
)
return self._kv_indices_scratch[:required_tokens]
def _asm_context_prefill_indices(
self, forward_batch: ForwardBatch, bs: int, num_kv_slots: int
):
"""KV slots and cu_seqlens_k for the ASM context-chunk prefill, computed
once per batch and shared by every full-attention layer.
For a plain extend batch AiterIndicesUpdaterPrefill lays kv_indices out
token by token with kv_indptr = cumsum(seq_lens), so the slots to gather
are the first sum(seq_lens) entries and cu_seqlens_k is kv_indptr itself;
both follow from host-side lengths without a device sync. Anything else
(spec batches, missing host lengths, or a short table) takes the generic
gather, which validates the metadata on the device.
"""
fm = self.forward_metadata
if fm.asm_ctx_ready:
return fm.asm_ctx_tok_idx, fm.asm_ctx_cu_k
fm.asm_ctx_ready = True
total_k = 0
if (
forward_batch.spec_info is None
and forward_batch.forward_mode.is_extend()
and forward_batch.seq_lens_cpu is not None
):
total_k = int(forward_batch.seq_lens_cpu[:bs].sum())
if 0 < total_k <= fm.kv_indices.numel():
tok_idx = fm.kv_indices[:total_k]
cu_k = fm.kv_indptr[: bs + 1]
if cu_k.dtype != torch.int32:
cu_k = cu_k.to(torch.int32)
if _GFX_ASM_CTX_GATHER_CHECK:
ref = _asm_context_prefill_gather_indices(
fm.kv_indptr[: bs + 1],
fm.kv_indices,
forward_batch.seq_lens[:bs],
num_kv_slots,
forward_batch.forward_mode,
)
assert (
ref is not None
and torch.equal(ref[0], tok_idx.to(torch.long))
and torch.equal(ref[1].to(torch.int32), cu_k)
), (
"asm context prefill: fast gather indices differ from the generic gather"
)
logger.info(
"[asm-context-prefill] fast gather indices verified: bs=%d total_k=%d",
bs,
total_k,
)
else:
gathered = _asm_context_prefill_gather_indices(
fm.kv_indptr[: bs + 1],
fm.kv_indices,
forward_batch.seq_lens[:bs],
num_kv_slots,
forward_batch.forward_mode,
)
if gathered is None:
return None, None
tok_idx, cu_k = gathered
cu_k = cu_k.to(torch.int32)
fm.asm_ctx_tok_idx, fm.asm_ctx_cu_k = tok_idx, cu_k
return tok_idx, cu_k
def _set_uniform_qo_indptr(
self, bs: int, tokens_per_req: int, device: torch.device
) -> torch.Tensor:
@@ -3473,6 +3549,8 @@ class AiterAttnBackend(AttentionBackend):
# faster; gathering the paged fp8 KV into a contiguous varlen
# buffer costs only ~20 us per layer at 70k context. The no-prefix
# first chunk already takes the ASM branch below.
# This applies to Qwen3.5 full-attention layers only currently,
# Other configurations fall through to the attention paths below.
if (
is_gfx95_supported()
and forward_batch.forward_mode.is_extend()
@@ -3492,15 +3570,10 @@ class AiterAttnBackend(AttentionBackend):
):
bs = forward_batch.batch_size
k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id)
gathered = _asm_context_prefill_gather_indices(
self.forward_metadata.kv_indptr[: bs + 1],
self.forward_metadata.kv_indices,
forward_batch.seq_lens[:bs],
self.token_to_kv_pool.get_key_buffer(layer.layer_id).shape[0],
forward_batch.forward_mode,
tok_idx, cu_k = self._asm_context_prefill_indices(
forward_batch, bs, k_cache.shape[0]
)
if gathered is not None:
tok_idx, cu_k = gathered
if tok_idx is not None:
hk = layer.tp_k_head_num * layer.qk_head_dim
hv = layer.tp_v_head_num * layer.v_head_dim
# uint8 view: index_select is not implemented for fp8.
@@ -3531,7 +3604,7 @@ class AiterAttnBackend(AttentionBackend):
k_descale.reshape(1),
v_descale.reshape(1),
self.qo_indptr[:bs0],
cu_k.to(torch.int32),
cu_k,
self.forward_metadata.max_q_len,
int(self.forward_metadata.max_kv_len),
softmax_scale=layer.scaling,