diff --git a/python/sglang/srt/layers/rocm_linear_utils.py b/python/sglang/srt/layers/rocm_linear_utils.py index 74c180226f..ba56fd6cd1 100644 --- a/python/sglang/srt/layers/rocm_linear_utils.py +++ b/python/sglang/srt/layers/rocm_linear_utils.py @@ -3,7 +3,24 @@ from aiter.ops.triton.fused_kv_cache import fused_qk_rope_cat_and_cache_mla from aiter.ops.triton.fused_qk_concat import fused_qk_rope_cat from aiter.tuned_gemm import tgemm -__all__ = ["fused_qk_rope_cat", "fused_qk_rope_cat_and_cache_mla"] +from sglang.srt.utils import is_gfx95_supported + +__all__ = [ + "fused_fp8_bmm_rope_cat_and_cache_mla", + "fused_qk_rope_cat", + "fused_qk_rope_cat_and_cache_mla", +] + +# This module is imported wherever AITER is on, gfx942 included, but the fused +# bmm+rope+cache op is gfx95-only. Import it behind the same predicate its one +# caller gates on, so an aiter build without the op cannot take down every +# DeepSeek import on another card. The name stays bound either way. +if is_gfx95_supported(): + from aiter.ops.triton.fusions.fused_bmm_rope_kv_cache import ( + fused_fp8_bmm_rope_cat_and_cache_mla, + ) +else: + fused_fp8_bmm_rope_cat_and_cache_mla = None def aiter_dsv3_router_gemm( diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py index fe112743a3..0f277b183e 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py @@ -138,7 +138,10 @@ if _use_aiter_gfx95: fused_flatten_mxfp4_quant, fused_rms_mxfp4_quant, ) - from sglang.srt.layers.rocm_linear_utils import fused_qk_rope_cat_and_cache_mla + from sglang.srt.layers.rocm_linear_utils import ( + fused_fp8_bmm_rope_cat_and_cache_mla, + fused_qk_rope_cat_and_cache_mla, + ) def _absorb_weight_bf16(w: torch.Tensor, w_scale) -> torch.Tensor: @@ -374,6 +377,74 @@ def _fused_rope_cat_and_cache( ) +def _can_fuse_bmm_rope_cat_and_cache(attn: DeepseekV2AttentionMLA) -> bool: + """Whether one AITER kernel can do the q absorb, the RoPE and the KV write. + + Those are otherwise two launches -- ``rocm_absorb_q_bmm`` in prepare and + ``_fused_rope_cat_and_cache`` in core -- and at decode shapes each fills + well under half the CUs, so a fused grid runs them side by side instead of + back to back. Every term below mirrors a branch one of those two would + otherwise have taken, so the fused path is only chosen where it is exactly + equivalent. + + The caller adds one more term it alone can see: DCP q replication splits the + absorb across a different weight, which this kernel does not carry. + + DCP is excluded outright: its decode phase all-gathers q_nope_out and q_pe + before core runs, and this path has no q_nope_out to hand it -- the absorb + has not happened yet. + """ + return ( + _use_aiter_gfx95 + and not get_parallel().dcp_enabled + and attn.w_kc.dtype == torch.float8_e4m3fn + and attn.rotary_emb is not None + and attn._skip_rope_for_dsa_tilelang_fused() + and attn.current_attention_backend in FORWARD_ABSORB_CORE_ATTENTION_BACKENDS + and not attn.use_deep_gemm_bmm + and not _SGLANG_EXPERIMENTAL_LORA_OPTI + and not is_kv_b_lora_active(attn) + ) + + +def _fused_bmm_rope_cat_and_cache( + attn: DeepseekV2AttentionMLA, + q_nope: torch.Tensor, + q_pe: torch.Tensor, + k_nope: torch.Tensor, + k_pe: torch.Tensor, + positions: torch.Tensor, + out_cache_loc: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """q absorb + RoPE + concat + KV-cache write in one AITER kernel. + + Takes q_nope un-absorbed: the BMM ``rocm_absorb_q_bmm`` would have done is + the first half of this kernel's grid. + """ + kv_cache_dtype = fp8_dtype if attn.kv_cache_dtype == "fp8_e4m3" else q_nope.dtype + # Same weights and group size the fp8 branch of rocm_absorb_q_bmm passes to + # batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant; + # that call takes q_nope as (B, QH, P) and transposes internally, while this + # kernel wants (QH, B, P) up front. + return fused_fp8_bmm_rope_cat_and_cache_mla( + q_nope.transpose(0, 1), + attn.w_kc.transpose(-1, -2), + attn.w_scale, + q_pe, + k_nope, + k_pe, + get_token_to_kv_pool().get_key_buffer(attn.attn_mqa.layer_id), + out_cache_loc, + positions, + attn.rotary_emb.cos_cache, + attn.rotary_emb.sin_cache, + group_size=128, + k_scale=attn.attn_mqa.k_scale, + is_neox=attn.rotary_emb.is_neox_style, + q_out_dtype=kv_cache_dtype, + ) + + class DeepseekMLARocmForwardMixin: def forward_absorb_rocm_prepare( self: DeepseekV2AttentionMLA, @@ -546,6 +617,10 @@ class DeepseekMLARocmForwardMixin: q_nope, q_pe, k_pe = self._split_q_nope_pe(q, latent_cache) + fuse_bmm_rope_cache = not q_replicate_active and ( + _can_fuse_bmm_rope_cat_and_cache(self) + ) + if q_replicate_active: q_nope_out = ( torch.bmm(q_nope.transpose(0, 1), self.w_kc_qrep) @@ -583,20 +658,25 @@ class DeepseekMLARocmForwardMixin: expected_m, ) q_nope_out = q_nope_out[:, :expected_m, :] + elif fuse_bmm_rope_cache: + # The absorb is the first half of the kernel core will run, and + # that kernel wants q_nope un-absorbed. Nothing to do here. + q_nope_out = None else: q_nope_out = rocm_absorb_q_bmm( self, q_nope, is_capture_mode=get_is_capture_mode() ) - q_nope_out = q_nope_out.transpose(0, 1) - if _SGLANG_EXPERIMENTAL_LORA_OPTI: - from sglang.srt.lora.trtllm_lora_temp.deepseek_mla_correction import ( - kv_b_lora_q_apply, - ) + if q_nope_out is not None: + q_nope_out = q_nope_out.transpose(0, 1) + if _SGLANG_EXPERIMENTAL_LORA_OPTI: + from sglang.srt.lora.trtllm_lora_temp.deepseek_mla_correction import ( + kv_b_lora_q_apply, + ) - q_nope_out = kv_b_lora_q_apply(self, q_nope, q_nope_out, _kvb_q) - elif is_kv_b_lora_active(self): - q_nope_out = apply_kv_b_lora_q_correction(self, q_nope, q_nope_out) + q_nope_out = kv_b_lora_q_apply(self, q_nope, q_nope_out, _kvb_q) + elif is_kv_b_lora_active(self): + q_nope_out = apply_kv_b_lora_q_correction(self, q_nope, q_nope_out) fuse_rope_for_trtllm_mla = self._fuse_rope_for_trtllm_mla(forward_batch) @@ -672,6 +752,7 @@ class DeepseekMLARocmForwardMixin: positions, topk_indices, llama_4_scaling, + q_nope if fuse_bmm_rope_cache else None, ) def forward_absorb_rocm_core( @@ -685,20 +766,35 @@ class DeepseekMLARocmForwardMixin: positions, topk_indices, llama_4_scaling, + q_nope_unabsorbed=None, ): save_kv_cache = True if self.current_attention_backend in FORWARD_ABSORB_CORE_ATTENTION_BACKENDS: if self._skip_rope_for_dsa_tilelang_fused() and self.rotary_emb is not None: - q_cat, _, k_pe_fused, _ = _fused_rope_cat_and_cache( - self, - q_nope_out, - q_pe, - k_nope, - k_pe, - positions, - forward_batch.out_cache_loc, - ) + if q_nope_unabsorbed is not None: + # prepare left the absorb to us: one kernel for the BMM, + # the RoPE and the KV write instead of two launches that + # each leave most of the machine idle. + q_cat, _, k_pe_fused, _ = _fused_bmm_rope_cat_and_cache( + self, + q_nope_unabsorbed, + q_pe, + k_nope, + k_pe, + positions, + forward_batch.out_cache_loc, + ) + else: + q_cat, _, k_pe_fused, _ = _fused_rope_cat_and_cache( + self, + q_nope_out, + q_pe, + k_nope, + k_pe, + positions, + forward_batch.out_cache_loc, + ) save_kv_cache = False # Pass q_cat straight to attn_mqa with q_rope=None so the backend # reuses it as a zero-copy view instead of rebuilding a tensor diff --git a/test/registered/unit/models/test_rocm_mla_absorb_fusion_gate.py b/test/registered/unit/models/test_rocm_mla_absorb_fusion_gate.py new file mode 100644 index 0000000000..71325cd056 --- /dev/null +++ b/test/registered/unit/models/test_rocm_mla_absorb_fusion_gate.py @@ -0,0 +1,94 @@ +"""The gate that picks the fused q-absorb + RoPE + KV-write kernel. + +Every term names a branch that one of the two kernels it replaces would +otherwise have taken, so the fused path is only equivalent where all of them +hold. A term quietly dropped here would route a shape the kernel does not +implement, and the wrong answer would look like a plausible one. + +The caller carries one further term, ``not q_replicate_active``, which needs a +ForwardBatch to evaluate and so is not part of this predicate. +""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.models.deepseek_common.attention_forward_methods import ( + forward_mla_rocm, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +_CAN_FUSE = forward_mla_rocm._can_fuse_bmm_rope_cat_and_cache + + +def _attn(**overrides): + """An attention module every term accepts, before the caller breaks one.""" + attn = SimpleNamespace( + w_kc=SimpleNamespace(dtype=torch.float8_e4m3fn), + rotary_emb=object(), + current_attention_backend="dsa", + use_deep_gemm_bmm=False, + kv_b_proj=SimpleNamespace(set_lora=False), + _skip_rope_for_dsa_tilelang_fused=lambda: True, + ) + for k, v in overrides.items(): + setattr(attn, k, v) + return attn + + +def _patch_dcp(case, *, dcp_enabled: bool): + """Give the gate a parallel context; the real one needs an initialized runtime.""" + parallel = SimpleNamespace(dcp_enabled=dcp_enabled) + saved = forward_mla_rocm.get_parallel + forward_mla_rocm.get_parallel = lambda: parallel + case.addCleanup(setattr, forward_mla_rocm, "get_parallel", saved) + return parallel + + +class TestFusedAbsorbGate(CustomTestCase): + def setUp(self): + # The first term is the platform, and it short-circuits everything else; + # patch it so the remaining terms are reachable off gfx95. + self._saved = forward_mla_rocm._use_aiter_gfx95 + forward_mla_rocm._use_aiter_gfx95 = True + self.addCleanup(setattr, forward_mla_rocm, "_use_aiter_gfx95", self._saved) + self._parallel = _patch_dcp(self, dcp_enabled=False) + + def test_dcp_turns_it_off(self): + """DCP decode all-gathers q_nope_out, which this path never produces. + + Without this term the fused branch sets q_nope_out to None and + all_gather_q_for_mla_decode dereferences it. + """ + self._parallel.dcp_enabled = True + self.assertFalse(_CAN_FUSE(_attn())) + + def test_takes_the_fused_path_when_every_term_holds(self): + self.assertTrue(_CAN_FUSE(_attn())) + + def test_off_gfx95_nothing_else_is_consulted(self): + forward_mla_rocm._use_aiter_gfx95 = False + # An attn that would otherwise qualify, so only the platform can decide. + self.assertFalse(_CAN_FUSE(_attn())) + + def test_each_term_alone_turns_it_off(self): + # The kernel absorbs an fp8 weight, needs a rope to fuse, writes the + # tilelang DSA cache layout, and has no LoRA or deep-gemm variant. + for name, broken in ( + ("w_kc", SimpleNamespace(dtype=torch.bfloat16)), + ("rotary_emb", None), + ("current_attention_backend", "triton"), + ("use_deep_gemm_bmm", True), + ("kv_b_proj", SimpleNamespace(set_lora=True)), + ("_skip_rope_for_dsa_tilelang_fused", lambda: False), + ): + with self.subTest(term=name): + self.assertFalse(_CAN_FUSE(_attn(**{name: broken}))) + + +if __name__ == "__main__": + unittest.main()