[ROCm] Fuse the MLA q absorb into the RoPE + KV-write kernel on gfx950 (#38340)

Co-authored-by: JohnQinAMD <zhengsheng1997@gmail.com>
Co-authored-by: RuibinCheung <ruibzhan@amd.com>
This commit is contained in:
xiaobochen-amd
2026-09-23 02:07:54 -07:00
committed by GitHub
co-authored by JohnQinAMD RuibinCheung
parent 401d5aedf3
commit a89f849158
3 changed files with 226 additions and 19 deletions
+18 -1
View File
@@ -3,7 +3,24 @@ from aiter.ops.triton.fused_kv_cache import fused_qk_rope_cat_and_cache_mla
from aiter.ops.triton.fused_qk_concat import fused_qk_rope_cat
from aiter.tuned_gemm import tgemm
__all__ = ["fused_qk_rope_cat", "fused_qk_rope_cat_and_cache_mla"]
from sglang.srt.utils import is_gfx95_supported
__all__ = [
"fused_fp8_bmm_rope_cat_and_cache_mla",
"fused_qk_rope_cat",
"fused_qk_rope_cat_and_cache_mla",
]
# This module is imported wherever AITER is on, gfx942 included, but the fused
# bmm+rope+cache op is gfx95-only. Import it behind the same predicate its one
# caller gates on, so an aiter build without the op cannot take down every
# DeepSeek import on another card. The name stays bound either way.
if is_gfx95_supported():
from aiter.ops.triton.fusions.fused_bmm_rope_kv_cache import (
fused_fp8_bmm_rope_cat_and_cache_mla,
)
else:
fused_fp8_bmm_rope_cat_and_cache_mla = None
def aiter_dsv3_router_gemm(
@@ -138,7 +138,10 @@ if _use_aiter_gfx95:
fused_flatten_mxfp4_quant,
fused_rms_mxfp4_quant,
)
from sglang.srt.layers.rocm_linear_utils import fused_qk_rope_cat_and_cache_mla
from sglang.srt.layers.rocm_linear_utils import (
fused_fp8_bmm_rope_cat_and_cache_mla,
fused_qk_rope_cat_and_cache_mla,
)
def _absorb_weight_bf16(w: torch.Tensor, w_scale) -> torch.Tensor:
@@ -374,6 +377,74 @@ def _fused_rope_cat_and_cache(
)
def _can_fuse_bmm_rope_cat_and_cache(attn: DeepseekV2AttentionMLA) -> bool:
"""Whether one AITER kernel can do the q absorb, the RoPE and the KV write.
Those are otherwise two launches -- ``rocm_absorb_q_bmm`` in prepare and
``_fused_rope_cat_and_cache`` in core -- and at decode shapes each fills
well under half the CUs, so a fused grid runs them side by side instead of
back to back. Every term below mirrors a branch one of those two would
otherwise have taken, so the fused path is only chosen where it is exactly
equivalent.
The caller adds one more term it alone can see: DCP q replication splits the
absorb across a different weight, which this kernel does not carry.
DCP is excluded outright: its decode phase all-gathers q_nope_out and q_pe
before core runs, and this path has no q_nope_out to hand it -- the absorb
has not happened yet.
"""
return (
_use_aiter_gfx95
and not get_parallel().dcp_enabled
and attn.w_kc.dtype == torch.float8_e4m3fn
and attn.rotary_emb is not None
and attn._skip_rope_for_dsa_tilelang_fused()
and attn.current_attention_backend in FORWARD_ABSORB_CORE_ATTENTION_BACKENDS
and not attn.use_deep_gemm_bmm
and not _SGLANG_EXPERIMENTAL_LORA_OPTI
and not is_kv_b_lora_active(attn)
)
def _fused_bmm_rope_cat_and_cache(
attn: DeepseekV2AttentionMLA,
q_nope: torch.Tensor,
q_pe: torch.Tensor,
k_nope: torch.Tensor,
k_pe: torch.Tensor,
positions: torch.Tensor,
out_cache_loc: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""q absorb + RoPE + concat + KV-cache write in one AITER kernel.
Takes q_nope un-absorbed: the BMM ``rocm_absorb_q_bmm`` would have done is
the first half of this kernel's grid.
"""
kv_cache_dtype = fp8_dtype if attn.kv_cache_dtype == "fp8_e4m3" else q_nope.dtype
# Same weights and group size the fp8 branch of rocm_absorb_q_bmm passes to
# batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant;
# that call takes q_nope as (B, QH, P) and transposes internally, while this
# kernel wants (QH, B, P) up front.
return fused_fp8_bmm_rope_cat_and_cache_mla(
q_nope.transpose(0, 1),
attn.w_kc.transpose(-1, -2),
attn.w_scale,
q_pe,
k_nope,
k_pe,
get_token_to_kv_pool().get_key_buffer(attn.attn_mqa.layer_id),
out_cache_loc,
positions,
attn.rotary_emb.cos_cache,
attn.rotary_emb.sin_cache,
group_size=128,
k_scale=attn.attn_mqa.k_scale,
is_neox=attn.rotary_emb.is_neox_style,
q_out_dtype=kv_cache_dtype,
)
class DeepseekMLARocmForwardMixin:
def forward_absorb_rocm_prepare(
self: DeepseekV2AttentionMLA,
@@ -546,6 +617,10 @@ class DeepseekMLARocmForwardMixin:
q_nope, q_pe, k_pe = self._split_q_nope_pe(q, latent_cache)
fuse_bmm_rope_cache = not q_replicate_active and (
_can_fuse_bmm_rope_cat_and_cache(self)
)
if q_replicate_active:
q_nope_out = (
torch.bmm(q_nope.transpose(0, 1), self.w_kc_qrep)
@@ -583,20 +658,25 @@ class DeepseekMLARocmForwardMixin:
expected_m,
)
q_nope_out = q_nope_out[:, :expected_m, :]
elif fuse_bmm_rope_cache:
# The absorb is the first half of the kernel core will run, and
# that kernel wants q_nope un-absorbed. Nothing to do here.
q_nope_out = None
else:
q_nope_out = rocm_absorb_q_bmm(
self, q_nope, is_capture_mode=get_is_capture_mode()
)
q_nope_out = q_nope_out.transpose(0, 1)
if _SGLANG_EXPERIMENTAL_LORA_OPTI:
from sglang.srt.lora.trtllm_lora_temp.deepseek_mla_correction import (
kv_b_lora_q_apply,
)
if q_nope_out is not None:
q_nope_out = q_nope_out.transpose(0, 1)
if _SGLANG_EXPERIMENTAL_LORA_OPTI:
from sglang.srt.lora.trtllm_lora_temp.deepseek_mla_correction import (
kv_b_lora_q_apply,
)
q_nope_out = kv_b_lora_q_apply(self, q_nope, q_nope_out, _kvb_q)
elif is_kv_b_lora_active(self):
q_nope_out = apply_kv_b_lora_q_correction(self, q_nope, q_nope_out)
q_nope_out = kv_b_lora_q_apply(self, q_nope, q_nope_out, _kvb_q)
elif is_kv_b_lora_active(self):
q_nope_out = apply_kv_b_lora_q_correction(self, q_nope, q_nope_out)
fuse_rope_for_trtllm_mla = self._fuse_rope_for_trtllm_mla(forward_batch)
@@ -672,6 +752,7 @@ class DeepseekMLARocmForwardMixin:
positions,
topk_indices,
llama_4_scaling,
q_nope if fuse_bmm_rope_cache else None,
)
def forward_absorb_rocm_core(
@@ -685,20 +766,35 @@ class DeepseekMLARocmForwardMixin:
positions,
topk_indices,
llama_4_scaling,
q_nope_unabsorbed=None,
):
save_kv_cache = True
if self.current_attention_backend in FORWARD_ABSORB_CORE_ATTENTION_BACKENDS:
if self._skip_rope_for_dsa_tilelang_fused() and self.rotary_emb is not None:
q_cat, _, k_pe_fused, _ = _fused_rope_cat_and_cache(
self,
q_nope_out,
q_pe,
k_nope,
k_pe,
positions,
forward_batch.out_cache_loc,
)
if q_nope_unabsorbed is not None:
# prepare left the absorb to us: one kernel for the BMM,
# the RoPE and the KV write instead of two launches that
# each leave most of the machine idle.
q_cat, _, k_pe_fused, _ = _fused_bmm_rope_cat_and_cache(
self,
q_nope_unabsorbed,
q_pe,
k_nope,
k_pe,
positions,
forward_batch.out_cache_loc,
)
else:
q_cat, _, k_pe_fused, _ = _fused_rope_cat_and_cache(
self,
q_nope_out,
q_pe,
k_nope,
k_pe,
positions,
forward_batch.out_cache_loc,
)
save_kv_cache = False
# Pass q_cat straight to attn_mqa with q_rope=None so the backend
# reuses it as a zero-copy view instead of rebuilding a tensor