[AMD] Reuse KV gather indices across ASM context prefill layers (#39901)
Co-authored-by: Zijie Chen <300606707+zijiecode@users.noreply.github.com> Co-authored-by: jacky.cheng <yichiche@amd.com>
This commit is contained in:
co-authored by
Zijie Chen
jacky.cheng
parent
5c18780c31
commit
720617bb5b
@@ -7,6 +7,7 @@ end to end attention solution with aiter kernels
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
@@ -158,6 +159,11 @@ class ForwardMetadata:
|
||||
swa_out_cache_loc: Optional[torch.Tensor] = None
|
||||
local_kv_lens: Optional[torch.Tensor] = None
|
||||
verify_token_table: Optional[torch.Tensor] = None
|
||||
# ASM context-chunk prefill: KV slots to gather and cu_seqlens_k, computed
|
||||
# once per batch by AiterAttnBackend._asm_context_prefill_indices.
|
||||
asm_ctx_ready: bool = False
|
||||
asm_ctx_tok_idx: Optional[torch.Tensor] = None
|
||||
asm_ctx_cu_k: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
_AITER_PARTITION_SIZE_ROCM = 256
|
||||
@@ -178,6 +184,12 @@ def _aiter_fp8_asm_supports_gqa(num_q_heads: int, num_kv_heads: int) -> bool:
|
||||
return (num_q_heads // num_kv_heads) in _AITER_FP8_ASM_GQA_RATIOS
|
||||
|
||||
|
||||
# Cross-check the per-batch fast indices against the generic gather (syncs).
|
||||
_GFX_ASM_CTX_GATHER_CHECK = (
|
||||
os.environ.get("SGLANG_GFX_ASM_CTX_GATHER_CHECK", "0") == "1"
|
||||
)
|
||||
|
||||
|
||||
def _asm_context_prefill_gather_indices(
|
||||
kv_indptr: torch.Tensor,
|
||||
kv_indices: torch.Tensor,
|
||||
@@ -1059,6 +1071,70 @@ class AiterAttnBackend(AttentionBackend):
|
||||
)
|
||||
return self._kv_indices_scratch[:required_tokens]
|
||||
|
||||
def _asm_context_prefill_indices(
|
||||
self, forward_batch: ForwardBatch, bs: int, num_kv_slots: int
|
||||
):
|
||||
"""KV slots and cu_seqlens_k for the ASM context-chunk prefill, computed
|
||||
once per batch and shared by every full-attention layer.
|
||||
|
||||
For a plain extend batch AiterIndicesUpdaterPrefill lays kv_indices out
|
||||
token by token with kv_indptr = cumsum(seq_lens), so the slots to gather
|
||||
are the first sum(seq_lens) entries and cu_seqlens_k is kv_indptr itself;
|
||||
both follow from host-side lengths without a device sync. Anything else
|
||||
(spec batches, missing host lengths, or a short table) takes the generic
|
||||
gather, which validates the metadata on the device.
|
||||
"""
|
||||
fm = self.forward_metadata
|
||||
if fm.asm_ctx_ready:
|
||||
return fm.asm_ctx_tok_idx, fm.asm_ctx_cu_k
|
||||
fm.asm_ctx_ready = True
|
||||
total_k = 0
|
||||
if (
|
||||
forward_batch.spec_info is None
|
||||
and forward_batch.forward_mode.is_extend()
|
||||
and forward_batch.seq_lens_cpu is not None
|
||||
):
|
||||
total_k = int(forward_batch.seq_lens_cpu[:bs].sum())
|
||||
if 0 < total_k <= fm.kv_indices.numel():
|
||||
tok_idx = fm.kv_indices[:total_k]
|
||||
cu_k = fm.kv_indptr[: bs + 1]
|
||||
if cu_k.dtype != torch.int32:
|
||||
cu_k = cu_k.to(torch.int32)
|
||||
if _GFX_ASM_CTX_GATHER_CHECK:
|
||||
ref = _asm_context_prefill_gather_indices(
|
||||
fm.kv_indptr[: bs + 1],
|
||||
fm.kv_indices,
|
||||
forward_batch.seq_lens[:bs],
|
||||
num_kv_slots,
|
||||
forward_batch.forward_mode,
|
||||
)
|
||||
assert (
|
||||
ref is not None
|
||||
and torch.equal(ref[0], tok_idx.to(torch.long))
|
||||
and torch.equal(ref[1].to(torch.int32), cu_k)
|
||||
), (
|
||||
"asm context prefill: fast gather indices differ from the generic gather"
|
||||
)
|
||||
logger.info(
|
||||
"[asm-context-prefill] fast gather indices verified: bs=%d total_k=%d",
|
||||
bs,
|
||||
total_k,
|
||||
)
|
||||
else:
|
||||
gathered = _asm_context_prefill_gather_indices(
|
||||
fm.kv_indptr[: bs + 1],
|
||||
fm.kv_indices,
|
||||
forward_batch.seq_lens[:bs],
|
||||
num_kv_slots,
|
||||
forward_batch.forward_mode,
|
||||
)
|
||||
if gathered is None:
|
||||
return None, None
|
||||
tok_idx, cu_k = gathered
|
||||
cu_k = cu_k.to(torch.int32)
|
||||
fm.asm_ctx_tok_idx, fm.asm_ctx_cu_k = tok_idx, cu_k
|
||||
return tok_idx, cu_k
|
||||
|
||||
def _set_uniform_qo_indptr(
|
||||
self, bs: int, tokens_per_req: int, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
@@ -3473,6 +3549,8 @@ class AiterAttnBackend(AttentionBackend):
|
||||
# faster; gathering the paged fp8 KV into a contiguous varlen
|
||||
# buffer costs only ~20 us per layer at 70k context. The no-prefix
|
||||
# first chunk already takes the ASM branch below.
|
||||
# This applies to Qwen3.5 full-attention layers only currently,
|
||||
# Other configurations fall through to the attention paths below.
|
||||
if (
|
||||
is_gfx95_supported()
|
||||
and forward_batch.forward_mode.is_extend()
|
||||
@@ -3492,15 +3570,10 @@ class AiterAttnBackend(AttentionBackend):
|
||||
):
|
||||
bs = forward_batch.batch_size
|
||||
k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id)
|
||||
gathered = _asm_context_prefill_gather_indices(
|
||||
self.forward_metadata.kv_indptr[: bs + 1],
|
||||
self.forward_metadata.kv_indices,
|
||||
forward_batch.seq_lens[:bs],
|
||||
self.token_to_kv_pool.get_key_buffer(layer.layer_id).shape[0],
|
||||
forward_batch.forward_mode,
|
||||
tok_idx, cu_k = self._asm_context_prefill_indices(
|
||||
forward_batch, bs, k_cache.shape[0]
|
||||
)
|
||||
if gathered is not None:
|
||||
tok_idx, cu_k = gathered
|
||||
if tok_idx is not None:
|
||||
hk = layer.tp_k_head_num * layer.qk_head_dim
|
||||
hv = layer.tp_v_head_num * layer.v_head_dim
|
||||
# uint8 view: index_select is not implemented for fp8.
|
||||
@@ -3531,7 +3604,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
k_descale.reshape(1),
|
||||
v_descale.reshape(1),
|
||||
self.qo_indptr[:bs0],
|
||||
cu_k.to(torch.int32),
|
||||
cu_k,
|
||||
self.forward_metadata.max_q_len,
|
||||
int(self.forward_metadata.max_kv_len),
|
||||
softmax_scale=layer.scaling,
|
||||
|
||||
Reference in New Issue
Block a user