[AMD] AITER MLA DCP decode ASM path (opt-in) (#40750)

This commit is contained in:
chien-an-chen
2026-09-30 22:47:55 -07:00
committed by GitHub
parent dc7defd294
commit 99c9d65b32
3 changed files with 208 additions and 18 deletions
+4
View File
@@ -944,6 +944,10 @@ class Envs:
# import and Triton cga_layout prerequisites hold. Set to 0 to force the
# zero-pad mla_decode_fwd fallback (benchmarking / emergency disable).
SGLANG_AITER_MLA_GLUON = EnvBool(True)
# Select the AITER MLA kernel for DCP decode. "gluon" preserves the native
# low-head Triton path; "asm" quantizes the DCP-gathered query to FP8 and
# runs mla_decode_fwd with persistent metadata and return_lse=True.
SGLANG_AITER_MLA_DCP_DECODE_BACKEND = EnvStr("gluon")
# DSV4 Aiter flags
SGLANG_OPT_USE_AITER_SILU_MUL = EnvBool(False)
@@ -325,6 +325,19 @@ class AiterAttnBackend(AttentionBackend):
self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA
self.dcp_world_size = get_parallel().attn_dcp_size
self.mla_dcp_decode_backend = (
envs.SGLANG_AITER_MLA_DCP_DECODE_BACKEND.get().lower()
)
if self.mla_dcp_decode_backend not in ("gluon", "asm"):
raise ValueError(
"SGLANG_AITER_MLA_DCP_DECODE_BACKEND must be 'gluon' or 'asm', "
f"got {self.mla_dcp_decode_backend!r}"
)
self.use_mla_dcp_asm = (
self.use_mla
and self.dcp_world_size > 1
and self.mla_dcp_decode_backend == "asm"
)
# Get v_head_dim based on model type
if self.use_mla:
@@ -567,9 +580,10 @@ class AiterAttnBackend(AttentionBackend):
intra_batch_mode = False
log_mla_gluon_capability(logger)
self.max_split_per_batch = 32 if _use_mla_ps_kernel else None
use_any_mla_persist = _use_mla_ps_kernel or self.use_mla_dcp_asm
self.max_split_per_batch = 32 if use_any_mla_persist else None
if self.num_draft_tokens is None and _use_mla_ps_kernel:
if self.num_draft_tokens is None and use_any_mla_persist:
self.max_split_per_batch = 64
self.fix_max_split_per_batch = self.max_split_per_batch
@@ -624,7 +638,33 @@ class AiterAttnBackend(AttentionBackend):
return "auto"
return "fp8_e4m3"
def make_mla_decode_meta_data_buffer(self, max_seqlen_qo, batch_size):
def _use_mla_decode_persist_metadata(self) -> bool:
"""Persist MLA metadata for non-DCP PS decode, or DCP ASM decode.
Matches main's ``_use_mla_ps_kernel and dcp_world_size <= 1`` when ASM
is off. DCP ASM needs the same buffers with fast_mode scheduling.
"""
return (_use_mla_ps_kernel and self.dcp_world_size <= 1) or (
self.use_mla_dcp_asm and self.dcp_world_size > 1
)
def _mla_decode_metadata_modes(self) -> tuple[bool, bool]:
if self.use_mla_dcp_asm and self.dcp_world_size > 1:
# Match the AITER/vLLM DCP path: persistent scheduling with the
# gathered-head tensor folded internally to qh16.
return True, False
return fast_mode, intra_batch_mode
def make_mla_decode_meta_data_buffer(
self,
max_seqlen_qo,
batch_size,
*,
metadata_fast_mode: Optional[bool] = None,
metadata_intra_batch_mode: Optional[bool] = None,
):
# Under DCP this is the gathered head count (num_head * dcp_world_size);
# equals num_head_padded when DCP is off.
nhead = self.mla_kernel_num_head_padded
dtype = self.kv_cache_dtype
@@ -636,6 +676,15 @@ class AiterAttnBackend(AttentionBackend):
(cu_num + batch_size - 1) // batch_size, self.fix_max_split_per_batch
)
metadata_fast_mode = (
fast_mode if metadata_fast_mode is None else metadata_fast_mode
)
metadata_intra_batch_mode = (
intra_batch_mode
if metadata_intra_batch_mode is None
else metadata_intra_batch_mode
)
(
(work_meta_data_size, work_meta_data_type),
(work_indptr_size, work_indptr_type),
@@ -650,9 +699,9 @@ class AiterAttnBackend(AttentionBackend):
dtype,
dtype,
is_sparse=False,
fast_mode=fast_mode,
fast_mode=metadata_fast_mode,
num_kv_splits=self.max_split_per_batch,
intra_batch_mode=intra_batch_mode,
intra_batch_mode=metadata_intra_batch_mode,
)
# aiter implementation
@@ -1343,6 +1392,52 @@ class AiterAttnBackend(AttentionBackend):
bs = fm.kv_indptr.shape[0] - 1
num_heads = layer.tp_q_head_num # gathered heads = num_local_heads * dcp
if self.use_mla_dcp_asm:
if fm.work_metadata is None:
raise RuntimeError(
"AITER MLA DCP ASM selected without persistent metadata"
)
q_mla = q.view(bs, num_heads, layer.qk_head_dim)
q_scale = k_descale
if q_mla.dtype != fp8_dtype:
q_mla, q_scale = scaled_fp8_quant(q_mla.reshape(bs, -1))
q_mla = q_mla.view(bs, num_heads, layer.qk_head_dim)
out = torch.empty(
(bs, num_heads, layer.v_head_dim),
dtype=self.input_dtype,
device=q.device,
)
_, lse = mla_decode_fwd(
q_mla,
k_buffer.view(-1, 1, 1, layer.qk_head_dim),
out,
fm.qo_indptr,
fm.kv_indptr[: bs + 1],
fm.kv_indices,
fm.kv_last_page_len,
fm.max_q_len or 1,
sm_scale=layer.scaling,
logit_cap=layer.logit_cap,
num_kv_splits=fm.num_kv_splits,
work_meta_data=fm.work_metadata,
work_indptr=fm.work_indptr,
work_info_set=fm.work_info_set,
reduce_indptr=fm.reduce_indptr,
reduce_final_map=fm.reduce_final_map,
reduce_partial_map=fm.reduce_partial_map,
q_scale=q_scale,
kv_scale=k_descale,
intra_batch_mode=False,
return_lse=True,
)
if lse is None:
raise RuntimeError(
"aiter mla_decode_fwd(return_lse=True) returned no LSE"
)
return out, lse.view(bs, num_heads)
out, lse = mla_gluon_decode(
q=q.view(bs, num_heads, layer.qk_head_dim),
k_buffer=k_buffer,
@@ -1658,9 +1753,10 @@ class AiterAttnBackend(AttentionBackend):
kv_last_page_len = self.kv_last_page_len[:bs]
max_q_len = 1
# DCP decode runs the aiter MLA kernel (builds its own block-table
# metadata in forward_decode), so skip the persist metadata.
if _use_mla_ps_kernel and self.dcp_world_size <= 1:
if self._use_mla_decode_persist_metadata():
metadata_fast_mode, metadata_intra_batch_mode = (
self._mla_decode_metadata_modes()
)
(
work_metadata,
work_indptr,
@@ -1668,7 +1764,12 @@ class AiterAttnBackend(AttentionBackend):
reduce_indptr,
reduce_final_map,
reduce_partial_map,
) = self.make_mla_decode_meta_data_buffer(max_q_len, bs)
) = self.make_mla_decode_meta_data_buffer(
max_q_len,
bs,
metadata_fast_mode=metadata_fast_mode,
metadata_intra_batch_mode=metadata_intra_batch_mode,
)
num_kv_splits = self.max_split_per_batch
@@ -1683,9 +1784,9 @@ class AiterAttnBackend(AttentionBackend):
reduce_final_map,
reduce_partial_map,
max_q_len,
fast_mode=fast_mode,
fast_mode=metadata_fast_mode,
max_split_per_batch=num_kv_splits,
intra_batch_mode=intra_batch_mode,
intra_batch_mode=metadata_intra_batch_mode,
)
self.forward_metadata = ForwardMetadata(
@@ -2327,11 +2428,14 @@ class AiterAttnBackend(AttentionBackend):
)
# if self.use_mla and (_use_mla_ps_kernel or self.kv_cache_dtype == fp8_dtype):
if self.use_mla and _use_mla_ps_kernel:
if self.use_mla and (_use_mla_ps_kernel or self.use_mla_dcp_asm):
# for persistent mla_decode_fwd
max_seqlen_qo = (
1 if self.num_draft_tokens is None else self.num_draft_tokens
)
metadata_fast_mode, metadata_intra_batch_mode = (
(True, False) if self.use_mla_dcp_asm else (fast_mode, intra_batch_mode)
)
(
self.work_metadata,
@@ -2340,7 +2444,12 @@ class AiterAttnBackend(AttentionBackend):
self.reduce_indptr,
self.reduce_final_map,
self.reduce_partial_map,
) = self.make_mla_decode_meta_data_buffer(max_seqlen_qo, max_bs)
) = self.make_mla_decode_meta_data_buffer(
max_seqlen_qo,
max_bs,
metadata_fast_mode=metadata_fast_mode,
metadata_intra_batch_mode=metadata_intra_batch_mode,
)
else:
self.work_metadata = None
@@ -2500,9 +2609,10 @@ class AiterAttnBackend(AttentionBackend):
kv_last_page_len = self.cuda_graph_kv_last_page_len[:bs]
max_q_len = 1
# DCP decode builds its own block-table metadata in
# forward_decode, so the persist metadata is unused here.
if _use_mla_ps_kernel and self.dcp_world_size <= 1:
if self._use_mla_decode_persist_metadata():
metadata_fast_mode, metadata_intra_batch_mode = (
self._mla_decode_metadata_modes()
)
num_kv_splits = self.max_split_per_batch
self.make_mla_meta_data(
@@ -2516,9 +2626,9 @@ class AiterAttnBackend(AttentionBackend):
self.reduce_final_map,
self.reduce_partial_map,
max_q_len,
fast_mode=fast_mode,
fast_mode=metadata_fast_mode,
max_split_per_batch=num_kv_splits,
intra_batch_mode=intra_batch_mode,
intra_batch_mode=metadata_intra_batch_mode,
)
work_metadata = self.work_metadata