[AMD] AITER MLA DCP decode ASM path (opt-in) (#40750)
This commit is contained in:
@@ -944,6 +944,10 @@ class Envs:
|
||||
# import and Triton cga_layout prerequisites hold. Set to 0 to force the
|
||||
# zero-pad mla_decode_fwd fallback (benchmarking / emergency disable).
|
||||
SGLANG_AITER_MLA_GLUON = EnvBool(True)
|
||||
# Select the AITER MLA kernel for DCP decode. "gluon" preserves the native
|
||||
# low-head Triton path; "asm" quantizes the DCP-gathered query to FP8 and
|
||||
# runs mla_decode_fwd with persistent metadata and return_lse=True.
|
||||
SGLANG_AITER_MLA_DCP_DECODE_BACKEND = EnvStr("gluon")
|
||||
|
||||
# DSV4 Aiter flags
|
||||
SGLANG_OPT_USE_AITER_SILU_MUL = EnvBool(False)
|
||||
|
||||
@@ -325,6 +325,19 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA
|
||||
|
||||
self.dcp_world_size = get_parallel().attn_dcp_size
|
||||
self.mla_dcp_decode_backend = (
|
||||
envs.SGLANG_AITER_MLA_DCP_DECODE_BACKEND.get().lower()
|
||||
)
|
||||
if self.mla_dcp_decode_backend not in ("gluon", "asm"):
|
||||
raise ValueError(
|
||||
"SGLANG_AITER_MLA_DCP_DECODE_BACKEND must be 'gluon' or 'asm', "
|
||||
f"got {self.mla_dcp_decode_backend!r}"
|
||||
)
|
||||
self.use_mla_dcp_asm = (
|
||||
self.use_mla
|
||||
and self.dcp_world_size > 1
|
||||
and self.mla_dcp_decode_backend == "asm"
|
||||
)
|
||||
|
||||
# Get v_head_dim based on model type
|
||||
if self.use_mla:
|
||||
@@ -567,9 +580,10 @@ class AiterAttnBackend(AttentionBackend):
|
||||
intra_batch_mode = False
|
||||
log_mla_gluon_capability(logger)
|
||||
|
||||
self.max_split_per_batch = 32 if _use_mla_ps_kernel else None
|
||||
use_any_mla_persist = _use_mla_ps_kernel or self.use_mla_dcp_asm
|
||||
self.max_split_per_batch = 32 if use_any_mla_persist else None
|
||||
|
||||
if self.num_draft_tokens is None and _use_mla_ps_kernel:
|
||||
if self.num_draft_tokens is None and use_any_mla_persist:
|
||||
self.max_split_per_batch = 64
|
||||
|
||||
self.fix_max_split_per_batch = self.max_split_per_batch
|
||||
@@ -624,7 +638,33 @@ class AiterAttnBackend(AttentionBackend):
|
||||
return "auto"
|
||||
return "fp8_e4m3"
|
||||
|
||||
def make_mla_decode_meta_data_buffer(self, max_seqlen_qo, batch_size):
|
||||
def _use_mla_decode_persist_metadata(self) -> bool:
|
||||
"""Persist MLA metadata for non-DCP PS decode, or DCP ASM decode.
|
||||
|
||||
Matches main's ``_use_mla_ps_kernel and dcp_world_size <= 1`` when ASM
|
||||
is off. DCP ASM needs the same buffers with fast_mode scheduling.
|
||||
"""
|
||||
return (_use_mla_ps_kernel and self.dcp_world_size <= 1) or (
|
||||
self.use_mla_dcp_asm and self.dcp_world_size > 1
|
||||
)
|
||||
|
||||
def _mla_decode_metadata_modes(self) -> tuple[bool, bool]:
|
||||
if self.use_mla_dcp_asm and self.dcp_world_size > 1:
|
||||
# Match the AITER/vLLM DCP path: persistent scheduling with the
|
||||
# gathered-head tensor folded internally to qh16.
|
||||
return True, False
|
||||
return fast_mode, intra_batch_mode
|
||||
|
||||
def make_mla_decode_meta_data_buffer(
|
||||
self,
|
||||
max_seqlen_qo,
|
||||
batch_size,
|
||||
*,
|
||||
metadata_fast_mode: Optional[bool] = None,
|
||||
metadata_intra_batch_mode: Optional[bool] = None,
|
||||
):
|
||||
# Under DCP this is the gathered head count (num_head * dcp_world_size);
|
||||
# equals num_head_padded when DCP is off.
|
||||
nhead = self.mla_kernel_num_head_padded
|
||||
dtype = self.kv_cache_dtype
|
||||
|
||||
@@ -636,6 +676,15 @@ class AiterAttnBackend(AttentionBackend):
|
||||
(cu_num + batch_size - 1) // batch_size, self.fix_max_split_per_batch
|
||||
)
|
||||
|
||||
metadata_fast_mode = (
|
||||
fast_mode if metadata_fast_mode is None else metadata_fast_mode
|
||||
)
|
||||
metadata_intra_batch_mode = (
|
||||
intra_batch_mode
|
||||
if metadata_intra_batch_mode is None
|
||||
else metadata_intra_batch_mode
|
||||
)
|
||||
|
||||
(
|
||||
(work_meta_data_size, work_meta_data_type),
|
||||
(work_indptr_size, work_indptr_type),
|
||||
@@ -650,9 +699,9 @@ class AiterAttnBackend(AttentionBackend):
|
||||
dtype,
|
||||
dtype,
|
||||
is_sparse=False,
|
||||
fast_mode=fast_mode,
|
||||
fast_mode=metadata_fast_mode,
|
||||
num_kv_splits=self.max_split_per_batch,
|
||||
intra_batch_mode=intra_batch_mode,
|
||||
intra_batch_mode=metadata_intra_batch_mode,
|
||||
)
|
||||
|
||||
# aiter implementation
|
||||
@@ -1343,6 +1392,52 @@ class AiterAttnBackend(AttentionBackend):
|
||||
bs = fm.kv_indptr.shape[0] - 1
|
||||
num_heads = layer.tp_q_head_num # gathered heads = num_local_heads * dcp
|
||||
|
||||
if self.use_mla_dcp_asm:
|
||||
if fm.work_metadata is None:
|
||||
raise RuntimeError(
|
||||
"AITER MLA DCP ASM selected without persistent metadata"
|
||||
)
|
||||
|
||||
q_mla = q.view(bs, num_heads, layer.qk_head_dim)
|
||||
q_scale = k_descale
|
||||
if q_mla.dtype != fp8_dtype:
|
||||
q_mla, q_scale = scaled_fp8_quant(q_mla.reshape(bs, -1))
|
||||
q_mla = q_mla.view(bs, num_heads, layer.qk_head_dim)
|
||||
|
||||
out = torch.empty(
|
||||
(bs, num_heads, layer.v_head_dim),
|
||||
dtype=self.input_dtype,
|
||||
device=q.device,
|
||||
)
|
||||
_, lse = mla_decode_fwd(
|
||||
q_mla,
|
||||
k_buffer.view(-1, 1, 1, layer.qk_head_dim),
|
||||
out,
|
||||
fm.qo_indptr,
|
||||
fm.kv_indptr[: bs + 1],
|
||||
fm.kv_indices,
|
||||
fm.kv_last_page_len,
|
||||
fm.max_q_len or 1,
|
||||
sm_scale=layer.scaling,
|
||||
logit_cap=layer.logit_cap,
|
||||
num_kv_splits=fm.num_kv_splits,
|
||||
work_meta_data=fm.work_metadata,
|
||||
work_indptr=fm.work_indptr,
|
||||
work_info_set=fm.work_info_set,
|
||||
reduce_indptr=fm.reduce_indptr,
|
||||
reduce_final_map=fm.reduce_final_map,
|
||||
reduce_partial_map=fm.reduce_partial_map,
|
||||
q_scale=q_scale,
|
||||
kv_scale=k_descale,
|
||||
intra_batch_mode=False,
|
||||
return_lse=True,
|
||||
)
|
||||
if lse is None:
|
||||
raise RuntimeError(
|
||||
"aiter mla_decode_fwd(return_lse=True) returned no LSE"
|
||||
)
|
||||
return out, lse.view(bs, num_heads)
|
||||
|
||||
out, lse = mla_gluon_decode(
|
||||
q=q.view(bs, num_heads, layer.qk_head_dim),
|
||||
k_buffer=k_buffer,
|
||||
@@ -1658,9 +1753,10 @@ class AiterAttnBackend(AttentionBackend):
|
||||
kv_last_page_len = self.kv_last_page_len[:bs]
|
||||
max_q_len = 1
|
||||
|
||||
# DCP decode runs the aiter MLA kernel (builds its own block-table
|
||||
# metadata in forward_decode), so skip the persist metadata.
|
||||
if _use_mla_ps_kernel and self.dcp_world_size <= 1:
|
||||
if self._use_mla_decode_persist_metadata():
|
||||
metadata_fast_mode, metadata_intra_batch_mode = (
|
||||
self._mla_decode_metadata_modes()
|
||||
)
|
||||
(
|
||||
work_metadata,
|
||||
work_indptr,
|
||||
@@ -1668,7 +1764,12 @@ class AiterAttnBackend(AttentionBackend):
|
||||
reduce_indptr,
|
||||
reduce_final_map,
|
||||
reduce_partial_map,
|
||||
) = self.make_mla_decode_meta_data_buffer(max_q_len, bs)
|
||||
) = self.make_mla_decode_meta_data_buffer(
|
||||
max_q_len,
|
||||
bs,
|
||||
metadata_fast_mode=metadata_fast_mode,
|
||||
metadata_intra_batch_mode=metadata_intra_batch_mode,
|
||||
)
|
||||
|
||||
num_kv_splits = self.max_split_per_batch
|
||||
|
||||
@@ -1683,9 +1784,9 @@ class AiterAttnBackend(AttentionBackend):
|
||||
reduce_final_map,
|
||||
reduce_partial_map,
|
||||
max_q_len,
|
||||
fast_mode=fast_mode,
|
||||
fast_mode=metadata_fast_mode,
|
||||
max_split_per_batch=num_kv_splits,
|
||||
intra_batch_mode=intra_batch_mode,
|
||||
intra_batch_mode=metadata_intra_batch_mode,
|
||||
)
|
||||
|
||||
self.forward_metadata = ForwardMetadata(
|
||||
@@ -2327,11 +2428,14 @@ class AiterAttnBackend(AttentionBackend):
|
||||
)
|
||||
|
||||
# if self.use_mla and (_use_mla_ps_kernel or self.kv_cache_dtype == fp8_dtype):
|
||||
if self.use_mla and _use_mla_ps_kernel:
|
||||
if self.use_mla and (_use_mla_ps_kernel or self.use_mla_dcp_asm):
|
||||
# for persistent mla_decode_fwd
|
||||
max_seqlen_qo = (
|
||||
1 if self.num_draft_tokens is None else self.num_draft_tokens
|
||||
)
|
||||
metadata_fast_mode, metadata_intra_batch_mode = (
|
||||
(True, False) if self.use_mla_dcp_asm else (fast_mode, intra_batch_mode)
|
||||
)
|
||||
|
||||
(
|
||||
self.work_metadata,
|
||||
@@ -2340,7 +2444,12 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.reduce_indptr,
|
||||
self.reduce_final_map,
|
||||
self.reduce_partial_map,
|
||||
) = self.make_mla_decode_meta_data_buffer(max_seqlen_qo, max_bs)
|
||||
) = self.make_mla_decode_meta_data_buffer(
|
||||
max_seqlen_qo,
|
||||
max_bs,
|
||||
metadata_fast_mode=metadata_fast_mode,
|
||||
metadata_intra_batch_mode=metadata_intra_batch_mode,
|
||||
)
|
||||
|
||||
else:
|
||||
self.work_metadata = None
|
||||
@@ -2500,9 +2609,10 @@ class AiterAttnBackend(AttentionBackend):
|
||||
kv_last_page_len = self.cuda_graph_kv_last_page_len[:bs]
|
||||
max_q_len = 1
|
||||
|
||||
# DCP decode builds its own block-table metadata in
|
||||
# forward_decode, so the persist metadata is unused here.
|
||||
if _use_mla_ps_kernel and self.dcp_world_size <= 1:
|
||||
if self._use_mla_decode_persist_metadata():
|
||||
metadata_fast_mode, metadata_intra_batch_mode = (
|
||||
self._mla_decode_metadata_modes()
|
||||
)
|
||||
num_kv_splits = self.max_split_per_batch
|
||||
|
||||
self.make_mla_meta_data(
|
||||
@@ -2516,9 +2626,9 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.reduce_final_map,
|
||||
self.reduce_partial_map,
|
||||
max_q_len,
|
||||
fast_mode=fast_mode,
|
||||
fast_mode=metadata_fast_mode,
|
||||
max_split_per_batch=num_kv_splits,
|
||||
intra_batch_mode=intra_batch_mode,
|
||||
intra_batch_mode=metadata_intra_batch_mode,
|
||||
)
|
||||
|
||||
work_metadata = self.work_metadata
|
||||
|
||||
Reference in New Issue
Block a user