[AMD][DCP 2/N] Enable aiter asm ps for kimi k3 prefill (#34438)
This commit is contained in:
@@ -134,6 +134,23 @@ class WrapperDispatch(Enum):
|
||||
CROSS_ATTENTION = auto()
|
||||
|
||||
|
||||
@dataclass
|
||||
class MlaPrefillPsMetadata:
|
||||
qo_indptr: torch.Tensor
|
||||
kv_indptr: torch.Tensor
|
||||
kv_indices: torch.Tensor
|
||||
work_metadata: torch.Tensor
|
||||
work_indptr: torch.Tensor
|
||||
work_info_set: torch.Tensor
|
||||
reduce_indptr: torch.Tensor
|
||||
reduce_final_map: torch.Tensor
|
||||
reduce_partial_map: torch.Tensor
|
||||
max_q_len: int
|
||||
is_causal: bool
|
||||
need_lse: bool
|
||||
num_partial_tiles: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class ForwardMetadata:
|
||||
kv_indptr: torch.Tensor
|
||||
@@ -153,7 +170,6 @@ class ForwardMetadata:
|
||||
custom_mask: Optional[torch.Tensor] = None
|
||||
mask_indptr: Optional[torch.Tensor] = None
|
||||
max_extend_len: Optional[int] = None
|
||||
fp8_prefill_kv_indices: Optional[torch.Tensor] = None
|
||||
swa_page_table: Optional[torch.Tensor] = None
|
||||
# full->SWA translated out_cache_loc (SWA KV-store write target)
|
||||
swa_out_cache_loc: Optional[torch.Tensor] = None
|
||||
@@ -166,6 +182,9 @@ class ForwardMetadata:
|
||||
asm_ctx_cu_k: Optional[torch.Tensor] = None
|
||||
# (page_indptr, page_ids, last_page_len); None means use the token-level table
|
||||
paged_kv_view: Optional[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = None
|
||||
prefill_ps_metadata: Optional[MlaPrefillPsMetadata] = None
|
||||
chunked_skip_prefix_ps_metadata: Optional[MlaPrefillPsMetadata] = None
|
||||
chunked_prefix_ps_metadatas: Optional[list[Optional[MlaPrefillPsMetadata]]] = None
|
||||
|
||||
|
||||
def _build_paged_kv_view(
|
||||
@@ -206,6 +225,10 @@ def _paged_prefill_asm_supports_gqa(num_q_heads: int, num_kv_heads: int) -> bool
|
||||
|
||||
_AITER_PARTITION_SIZE_ROCM = 256
|
||||
|
||||
# Query rows one asm-prefill work tile covers. The scheduler, the partial
|
||||
# buffers and mla_reduce_v1 must all agree on it.
|
||||
_PREFILL_TILE_Q = 256
|
||||
|
||||
|
||||
_DCP_VERIFY_TABLE_COLS_PER_BLOCK = 128
|
||||
|
||||
@@ -408,6 +431,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
(max_bs + 1,), dtype=torch.int64, device=model_runner.device
|
||||
)
|
||||
self._kv_indices_scratch: Optional[torch.Tensor] = None
|
||||
self._arange_buf: Optional[torch.Tensor] = None
|
||||
|
||||
# Create prefill indices updater
|
||||
if not skip_prefill:
|
||||
@@ -779,6 +803,14 @@ class AiterAttnBackend(AttentionBackend):
|
||||
dtype_kv=dtype,
|
||||
)
|
||||
|
||||
@property
|
||||
def _prefill_qlen_granularity(self) -> int:
|
||||
"""Query rows per scheduling unit: one tile's worth, divided across the
|
||||
query heads the kernel processes together."""
|
||||
return _PREFILL_TILE_Q // (
|
||||
self.fp8_prefill_num_head // self.fp8_prefill_num_kv_head
|
||||
)
|
||||
|
||||
def make_mla_prefill_ps_meta_data_buffer(
|
||||
self, batch_size: int, max_qlen: int, qlen_granularity: int
|
||||
):
|
||||
@@ -835,12 +867,12 @@ class AiterAttnBackend(AttentionBackend):
|
||||
reduce_final_map: torch.Tensor,
|
||||
reduce_partial_map: torch.Tensor,
|
||||
is_causal: bool = True,
|
||||
need_lse: bool = False,
|
||||
):
|
||||
gqa_ratio = self.fp8_prefill_num_head // self.fp8_prefill_num_kv_head
|
||||
num_heads_k = self.fp8_prefill_num_kv_head
|
||||
tile_q = 256
|
||||
qhead_granularity = gqa_ratio
|
||||
qlen_granularity = tile_q // qhead_granularity
|
||||
qlen_granularity = self._prefill_qlen_granularity
|
||||
kvlen_granularity = 128
|
||||
block_size = 1
|
||||
|
||||
@@ -865,6 +897,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
kvlen_granularity=kvlen_granularity,
|
||||
block_size=block_size,
|
||||
is_causal=is_causal,
|
||||
need_lse=need_lse,
|
||||
)
|
||||
|
||||
# for page size > 1 useful conversion function
|
||||
@@ -1158,6 +1191,16 @@ class AiterAttnBackend(AttentionBackend):
|
||||
)
|
||||
return self._kv_indices_scratch[:required_tokens]
|
||||
|
||||
def _get_arange(self, length: int) -> torch.Tensor:
|
||||
"""arange(length) grown to the next power of two."""
|
||||
if self._arange_buf is None or self._arange_buf.numel() < length:
|
||||
self._arange_buf = torch.arange(
|
||||
1 << max(length - 1, 0).bit_length(),
|
||||
device=self.device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
return self._arange_buf[:length]
|
||||
|
||||
def _asm_context_prefill_indices(
|
||||
self, forward_batch: ForwardBatch, bs: int, num_kv_slots: int
|
||||
):
|
||||
@@ -1508,6 +1551,20 @@ class AiterAttnBackend(AttentionBackend):
|
||||
v: torch.Tensor,
|
||||
layer: RadixAttention,
|
||||
):
|
||||
out, _ = self._mla_fp8_prefill_attn_ps(
|
||||
q, k, v, layer, self.forward_metadata.prefill_ps_metadata
|
||||
)
|
||||
return out
|
||||
|
||||
def _mla_fp8_prefill_attn_ps(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
layer: RadixAttention,
|
||||
ps: MlaPrefillPsMetadata,
|
||||
):
|
||||
"""Run the asm prefill over one PS metadata, returning (out, lse)."""
|
||||
total_q = q.shape[0]
|
||||
nhead = layer.tp_q_head_num
|
||||
v_head_dim = layer.v_head_dim
|
||||
@@ -1529,25 +1586,21 @@ class AiterAttnBackend(AttentionBackend):
|
||||
v = v.to(fp8_dtype)
|
||||
one_scale = torch.ones((), dtype=torch.float32, device=q.device)
|
||||
|
||||
tile_q = 256
|
||||
reduce_indptr = self.forward_metadata.reduce_indptr
|
||||
reduce_final_map = self.forward_metadata.reduce_final_map
|
||||
reduce_partial_map = self.forward_metadata.reduce_partial_map
|
||||
|
||||
partial_q = max(ps.num_partial_tiles, 1) * _PREFILL_TILE_Q
|
||||
logits = torch.empty(
|
||||
(reduce_partial_map.size(0) * tile_q, nhead, v_head_dim),
|
||||
(partial_q, nhead, v_head_dim),
|
||||
dtype=torch.float32,
|
||||
device=q.device,
|
||||
)
|
||||
attn_lse = torch.empty(
|
||||
(reduce_partial_map.size(0) * tile_q, nhead),
|
||||
(partial_q, nhead),
|
||||
dtype=torch.float32,
|
||||
device=q.device,
|
||||
)
|
||||
final_lse = torch.empty(
|
||||
(total_q, nhead),
|
||||
dtype=torch.float32,
|
||||
device=q.device,
|
||||
final_lse = (
|
||||
torch.empty((total_q, nhead), dtype=torch.float32, device=q.device)
|
||||
if ps.need_lse
|
||||
else None
|
||||
)
|
||||
output = q.new_empty(
|
||||
(total_q, nhead, v_head_dim),
|
||||
@@ -1558,14 +1611,14 @@ class AiterAttnBackend(AttentionBackend):
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
self.forward_metadata.qo_indptr,
|
||||
self.forward_metadata.kv_indptr,
|
||||
self.forward_metadata.fp8_prefill_kv_indices,
|
||||
self.forward_metadata.work_indptr,
|
||||
self.forward_metadata.work_info_set,
|
||||
self.forward_metadata.max_q_len,
|
||||
ps.qo_indptr,
|
||||
ps.kv_indptr,
|
||||
ps.kv_indices,
|
||||
ps.work_indptr,
|
||||
ps.work_info_set,
|
||||
ps.max_q_len,
|
||||
layer.scaling,
|
||||
True,
|
||||
ps.is_causal,
|
||||
logits,
|
||||
attn_lse,
|
||||
output,
|
||||
@@ -1576,16 +1629,23 @@ class AiterAttnBackend(AttentionBackend):
|
||||
mla_reduce_v1(
|
||||
logits,
|
||||
attn_lse,
|
||||
reduce_indptr,
|
||||
reduce_final_map,
|
||||
reduce_partial_map,
|
||||
tile_q,
|
||||
ps.reduce_indptr,
|
||||
ps.reduce_final_map,
|
||||
ps.reduce_partial_map,
|
||||
_PREFILL_TILE_Q,
|
||||
# Prefill PS metadata has no split cap; 0 keeps AITER's default reduce sizing.
|
||||
0,
|
||||
output,
|
||||
final_lse,
|
||||
)
|
||||
return output[:, : layer.tp_q_head_num, :] if head_pad else output
|
||||
if head_pad:
|
||||
output = output[:, : layer.tp_q_head_num, :]
|
||||
if final_lse is not None:
|
||||
# Slicing the padded head axis leaves the kernel's stride
|
||||
# behind, and merge_state reads its inputs as contiguous.
|
||||
output = output.contiguous()
|
||||
final_lse = final_lse[:, : layer.tp_q_head_num].contiguous()
|
||||
return output, final_lse
|
||||
|
||||
def _kv_index_blocks(self, bs: int) -> int:
|
||||
if self.max_context_len < _KV_INDEX_BLOCKS_MIN_CONTEXT:
|
||||
@@ -2092,48 +2152,21 @@ class AiterAttnBackend(AttentionBackend):
|
||||
qo_indptr = self.mla_indices_updater_prefill.qo_indptr
|
||||
kv_indptr = self.mla_indices_updater_prefill.kv_indptr
|
||||
|
||||
work_metadata = None
|
||||
work_indptr = None
|
||||
work_info_set = None
|
||||
reduce_indptr = None
|
||||
reduce_final_map = None
|
||||
reduce_partial_map = None
|
||||
fp8_prefill_kv_indices = None
|
||||
|
||||
# fp8 PS-ASM prefill memory-faults on gfx950 for 12-head
|
||||
# (zero-pad) models; keep it off and use flash-attn fallback.
|
||||
if self.use_fp8_prefill_attn and self.head_pad_mode != "zero":
|
||||
tile_q = 256
|
||||
qlen_granularity = tile_q // (
|
||||
self.fp8_prefill_num_head // self.fp8_prefill_num_kv_head
|
||||
)
|
||||
(
|
||||
work_metadata,
|
||||
work_indptr,
|
||||
work_info_set,
|
||||
reduce_indptr,
|
||||
reduce_final_map,
|
||||
reduce_partial_map,
|
||||
) = self.make_mla_prefill_ps_meta_data_buffer(
|
||||
bs, max_q_len, qlen_granularity
|
||||
)
|
||||
|
||||
self.make_mla_prefill_ps_meta_data(
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
forward_batch.seq_lens,
|
||||
work_metadata,
|
||||
work_indptr,
|
||||
work_info_set,
|
||||
reduce_indptr,
|
||||
reduce_final_map,
|
||||
reduce_partial_map,
|
||||
prefill_ps_metadata = None
|
||||
if self.use_fp8_prefill_attn:
|
||||
prefill_ps_metadata = self._build_prefill_ps_metadata(
|
||||
qo_indptr=qo_indptr,
|
||||
kv_indptr=kv_indptr,
|
||||
kv_lens_cpu=forward_batch.seq_lens_cpu,
|
||||
num_kv_tokens=forward_batch.seq_lens_sum,
|
||||
max_q_len=max_q_len,
|
||||
# The keys are the whole sequence, and the extend
|
||||
# tokens sit at its tail.
|
||||
is_causal=True,
|
||||
)
|
||||
|
||||
total_s = forward_batch.seq_lens_sum
|
||||
fp8_prefill_kv_indices = torch.arange(
|
||||
total_s, device=self.device, dtype=torch.int32
|
||||
need_lse=False,
|
||||
# Keep the static bound this path has always sized its
|
||||
# partial buffers with, rather than the emitted count.
|
||||
exact_partial_count=False,
|
||||
)
|
||||
|
||||
self.forward_metadata = ForwardMetadata(
|
||||
@@ -2143,13 +2176,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.kv_last_page_len[:bs],
|
||||
max_q_len,
|
||||
self.mla_indices_updater_prefill.max_kv_len,
|
||||
work_metadata=work_metadata,
|
||||
work_info_set=work_info_set,
|
||||
work_indptr=work_indptr,
|
||||
reduce_indptr=reduce_indptr,
|
||||
reduce_final_map=reduce_final_map,
|
||||
reduce_partial_map=reduce_partial_map,
|
||||
fp8_prefill_kv_indices=fp8_prefill_kv_indices,
|
||||
prefill_ps_metadata=prefill_ps_metadata,
|
||||
)
|
||||
else:
|
||||
self.indices_updater_prefill.update(
|
||||
@@ -2935,7 +2962,143 @@ class AiterAttnBackend(AttentionBackend):
|
||||
def init_mha_chunk_metadata(
|
||||
self, forward_batch: ForwardBatch, disable_flashinfer_ragged: bool = False
|
||||
) -> None:
|
||||
pass
|
||||
"""Build the chunked-prefix route's PS metadata, once per forward.
|
||||
|
||||
Its attentions each attend a subset of the sequence, while
|
||||
init_forward_metadata's PS metadata covers the whole sequence.
|
||||
"""
|
||||
# The one-shot core calls this hook too, with num_prefix_chunks 0.
|
||||
if not forward_batch.num_prefix_chunks or not self.use_fp8_prefill_attn:
|
||||
return
|
||||
|
||||
# Both passes query the extend tokens, so they share qo_indptr.
|
||||
extend_lens_cpu = torch.tensor(
|
||||
forward_batch.extend_seq_lens_cpu, dtype=torch.int32
|
||||
)
|
||||
qo_indptr_cpu = torch.zeros(len(extend_lens_cpu) + 1, dtype=torch.int32)
|
||||
qo_indptr_cpu[1:] = torch.cumsum(extend_lens_cpu, dim=0)
|
||||
qo_indptr = self.forward_metadata.qo_indptr
|
||||
max_q_len = self.forward_metadata.max_q_len
|
||||
|
||||
# disable_flashinfer_ragged asks for no ragged pass. The skip-prefix
|
||||
# pass is this backend's ragged one, so leave it on the varlen path.
|
||||
if not disable_flashinfer_ragged:
|
||||
self.forward_metadata.chunked_skip_prefix_ps_metadata = (
|
||||
self._build_prefill_ps_metadata(
|
||||
qo_indptr=qo_indptr,
|
||||
qo_indptr_cpu=qo_indptr_cpu,
|
||||
kv_indptr=qo_indptr,
|
||||
kv_indptr_cpu=qo_indptr_cpu,
|
||||
kv_lens_cpu=extend_lens_cpu,
|
||||
num_kv_tokens=int(qo_indptr_cpu[-1]),
|
||||
max_q_len=max_q_len,
|
||||
is_causal=True,
|
||||
need_lse=True,
|
||||
)
|
||||
)
|
||||
|
||||
metadatas: list[Optional[MlaPrefillPsMetadata]] = []
|
||||
for i in range(forward_batch.num_prefix_chunks):
|
||||
chunk_lens_cpu = forward_batch.prefix_chunk_seq_lens_cpu[i].to(torch.int32)
|
||||
# An empty kv range has no PS metadata shape.
|
||||
# _forward_extend_prefix_chunk falls back to varlen there.
|
||||
if forward_batch.prefix_chunk_has_zero_kv[i]:
|
||||
metadatas.append(None)
|
||||
continue
|
||||
chunk_indptr_cpu = torch.zeros(len(chunk_lens_cpu) + 1, dtype=torch.int32)
|
||||
chunk_indptr_cpu[1:] = torch.cumsum(chunk_lens_cpu, dim=0)
|
||||
metadatas.append(
|
||||
self._build_prefill_ps_metadata(
|
||||
qo_indptr=qo_indptr,
|
||||
qo_indptr_cpu=qo_indptr_cpu,
|
||||
kv_indptr=forward_batch.prefix_chunk_cu_seq_lens[i],
|
||||
kv_indptr_cpu=chunk_indptr_cpu,
|
||||
kv_lens_cpu=chunk_lens_cpu,
|
||||
num_kv_tokens=forward_batch.prefix_chunk_num_tokens[i],
|
||||
max_q_len=max_q_len,
|
||||
# Every key in a prefix chunk precedes every extend token.
|
||||
is_causal=False,
|
||||
need_lse=True,
|
||||
)
|
||||
)
|
||||
self.forward_metadata.chunked_prefix_ps_metadatas = metadatas
|
||||
|
||||
def _build_prefill_ps_metadata(
|
||||
self,
|
||||
*,
|
||||
qo_indptr: torch.Tensor,
|
||||
kv_indptr: torch.Tensor,
|
||||
kv_lens_cpu: torch.Tensor,
|
||||
num_kv_tokens: int,
|
||||
max_q_len: int,
|
||||
is_causal: bool,
|
||||
need_lse: bool,
|
||||
qo_indptr_cpu: Optional[torch.Tensor] = None,
|
||||
kv_indptr_cpu: Optional[torch.Tensor] = None,
|
||||
exact_partial_count: bool = True,
|
||||
) -> MlaPrefillPsMetadata:
|
||||
"""Plan one asm-prefill PS metadata over the key set the caller names."""
|
||||
(
|
||||
work_metadata,
|
||||
work_indptr,
|
||||
work_info_set,
|
||||
reduce_indptr,
|
||||
reduce_final_map,
|
||||
reduce_partial_map,
|
||||
) = self.make_mla_prefill_ps_meta_data_buffer(
|
||||
len(kv_lens_cpu), max_q_len, self._prefill_qlen_granularity
|
||||
)
|
||||
self.make_mla_prefill_ps_meta_data(
|
||||
qo_indptr if qo_indptr_cpu is None else qo_indptr_cpu,
|
||||
kv_indptr if kv_indptr_cpu is None else kv_indptr_cpu,
|
||||
kv_lens_cpu,
|
||||
work_metadata,
|
||||
work_indptr,
|
||||
work_info_set,
|
||||
reduce_indptr,
|
||||
reduce_final_map,
|
||||
reduce_partial_map,
|
||||
is_causal=is_causal,
|
||||
need_lse=need_lse,
|
||||
)
|
||||
if exact_partial_count:
|
||||
# One D2H sync per PS metadata, once per forward rather than per
|
||||
# layer. get_ps_metadata_v1 already pays several.
|
||||
num_partial_tiles = int(reduce_indptr[-1].item())
|
||||
if need_lse:
|
||||
assert num_partial_tiles > 0, (
|
||||
"the scheduler emitted no partial tiles, so mla_reduce_v1 "
|
||||
"would write no LSE for this partition"
|
||||
)
|
||||
if num_partial_tiles:
|
||||
max_partial_row = int(reduce_partial_map[:num_partial_tiles].max())
|
||||
assert (
|
||||
max_partial_row + _PREFILL_TILE_Q
|
||||
<= num_partial_tiles * _PREFILL_TILE_Q
|
||||
), (
|
||||
f"reduce_partial_map reaches row {max_partial_row} and the "
|
||||
f"tile is {_PREFILL_TILE_Q} rows, but the partial buffers "
|
||||
f"hold only {num_partial_tiles * _PREFILL_TILE_Q} rows"
|
||||
)
|
||||
else:
|
||||
num_partial_tiles = reduce_partial_map.size(0)
|
||||
return MlaPrefillPsMetadata(
|
||||
qo_indptr=qo_indptr,
|
||||
kv_indptr=kv_indptr,
|
||||
# The k/v handed to the kernel are contiguous and in key order, so
|
||||
# the page table is the identity.
|
||||
kv_indices=self._get_arange(num_kv_tokens),
|
||||
work_metadata=work_metadata,
|
||||
work_indptr=work_indptr,
|
||||
work_info_set=work_info_set,
|
||||
reduce_indptr=reduce_indptr,
|
||||
reduce_final_map=reduce_final_map,
|
||||
reduce_partial_map=reduce_partial_map,
|
||||
max_q_len=max_q_len,
|
||||
is_causal=is_causal,
|
||||
need_lse=need_lse,
|
||||
num_partial_tiles=num_partial_tiles,
|
||||
)
|
||||
|
||||
def _forward_extend_prefix_chunk(
|
||||
self,
|
||||
@@ -2946,6 +3109,10 @@ class AiterAttnBackend(AttentionBackend):
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
idx = forward_batch.prefix_chunk_idx
|
||||
ps_metadatas = self.forward_metadata.chunked_prefix_ps_metadatas
|
||||
if ps_metadatas is not None and ps_metadatas[idx] is not None:
|
||||
return self._mla_fp8_prefill_attn_ps(q, k, v, layer, ps_metadatas[idx])
|
||||
|
||||
output, lse = flash_attn_varlen_func(
|
||||
q,
|
||||
k,
|
||||
@@ -2958,6 +3125,8 @@ class AiterAttnBackend(AttentionBackend):
|
||||
causal=False,
|
||||
return_lse=True,
|
||||
)[:2]
|
||||
# the merge_state_triton needs lse layout [tokens, heads].
|
||||
# See https://github.com/sgl-project/sglang/blob/98fce73d5bd0a25afe7d68443d314190b1c47e64/python/sglang/kernels/ops/attention/merge_state.py#L13-L15
|
||||
return output, lse.transpose(0, 1).contiguous()
|
||||
|
||||
def _forward_extend_skip_prefix(
|
||||
@@ -2967,6 +3136,10 @@ class AiterAttnBackend(AttentionBackend):
|
||||
v: torch.Tensor,
|
||||
layer: RadixAttention,
|
||||
):
|
||||
ps = self.forward_metadata.chunked_skip_prefix_ps_metadata
|
||||
if ps is not None:
|
||||
return self._mla_fp8_prefill_attn_ps(q, k, v, layer, ps)
|
||||
|
||||
qo_indptr = self.forward_metadata.qo_indptr
|
||||
max_q_len = self.forward_metadata.max_q_len
|
||||
output, lse = flash_attn_varlen_func(
|
||||
@@ -2981,6 +3154,8 @@ class AiterAttnBackend(AttentionBackend):
|
||||
causal=True,
|
||||
return_lse=True,
|
||||
)[:2]
|
||||
# the merge_state_triton needs lse layout [tokens, heads].
|
||||
# See https://github.com/sgl-project/sglang/blob/98fce73d5bd0a25afe7d68443d314190b1c47e64/python/sglang/kernels/ops/attention/merge_state.py#L13-L15
|
||||
return output, lse.transpose(0, 1).contiguous()
|
||||
|
||||
@staticmethod
|
||||
@@ -3199,7 +3374,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
if forward_batch.mha_return_lse:
|
||||
return self._forward_extend_skip_prefix(q, k, v, layer)
|
||||
if self.dcp_world_size > 1:
|
||||
if self.use_fp8_prefill_attn and self.head_pad_mode != "zero":
|
||||
if self.use_fp8_prefill_attn:
|
||||
return self.mla_fp8_prefill_attn(q, k, v, layer)
|
||||
return flash_attn_varlen_func(
|
||||
q,
|
||||
@@ -3213,7 +3388,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
causal=True,
|
||||
)
|
||||
if kv_indices.shape[0] == 0 or extend_no_prefix:
|
||||
if self.use_fp8_prefill_attn and self.head_pad_mode != "zero":
|
||||
if self.use_fp8_prefill_attn:
|
||||
output = self.mla_fp8_prefill_attn(
|
||||
q,
|
||||
k,
|
||||
@@ -3247,7 +3422,6 @@ class AiterAttnBackend(AttentionBackend):
|
||||
|
||||
if (
|
||||
self.use_fp8_prefill_attn
|
||||
and self.head_pad_mode != "zero"
|
||||
and layer.kv_b_proj.weight.dtype == torch.uint8
|
||||
):
|
||||
# MXFP4 weights + FP8 prefill: fuse GEMM, nope/v split, and k_pe cat
|
||||
@@ -3287,7 +3461,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
== forward_batch.extend_seq_lens.shape
|
||||
)
|
||||
|
||||
if self.use_fp8_prefill_attn and self.head_pad_mode != "zero":
|
||||
if self.use_fp8_prefill_attn:
|
||||
return self.mla_fp8_prefill_attn(q, k, v, layer)
|
||||
else:
|
||||
return flash_attn_varlen_func(
|
||||
|
||||
Reference in New Issue
Block a user