[Perf] DeepSeek-V4.1: Run the small-batch decode fusions at TP2 (#43838)

Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com>
Co-authored-by: nbommi19 <227267412+nbommi19@users.noreply.github.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
weireweire
2026-10-19 21:17:05 -07:00
committed by GitHub
co-authored by weireweire nbommi19 Po-Han Huang Claude Opus 5.5 Po-Han Huang
parent fdd53418a0
commit 72d886efb8
9 changed files with 136 additions and 83 deletions
@@ -4119,7 +4119,7 @@ class DeepseekV4AttnBackend(
)
return swapab_attention(
q[..., :16, :],
q[..., : layer.tp_q_head_num, :],
swa_k_cache,
swa_page_indices,
swa_topk_lengths,
+9 -5
View File
@@ -31,7 +31,10 @@ from sglang.kernels.ops.attention.dsv4 import (
fused_rope_inplace,
)
from sglang.kernels.ops.attention.dsv4.wo_a import MAX_M as _FUSED_WO_A_MAX_TOKENS
from sglang.kernels.ops.attention.dsv4.wo_a import fused_rope_wo_a_bf16
from sglang.kernels.ops.attention.dsv4.wo_a import (
can_use_fused_wo_a,
fused_rope_wo_a_bf16,
)
from sglang.kernels.ops.attention.flash_mla_sm120 import SM120_DECODE_MAX_TOKENS
from sglang.kernels.ops.gemm.dsv4_wo_a import (
wo_a_bf16_gemv,
@@ -1365,7 +1368,7 @@ class MQALayer(MqaAttentionBase):
and not self.use_npu_arch35_mxfp8_wo_a
and self.wo_a.weight.dtype == torch.bfloat16
and self.wo_a.weight.shape == (self.n_local_groups * self.o_lora_rank, 4096)
and (self.n_local_groups, self.o_lora_rank) == (2, 1024)
and can_use_fused_wo_a(self.n_local_groups, self.o_lora_rank)
)
if _is_hip:
_hip.init_mqa_layer(self, quant_config)
@@ -3324,7 +3327,7 @@ class DeepseekV4DecoderLayerBase(nn.Module):
def _can_fuse_ffn_mhc(self) -> bool:
"""The MoE counterpart of `_can_fuse_attn_mhc`."""
return (
self.mlp.tp_size == 4
mhc.fused_mhc_tp_ok(self.mlp.tp_size)
and get_moe_a2a_backend().is_none()
and not self.mlp._shared_expert_tp1
and not should_skip_post_experts_all_reduce(is_tp_path=True)
@@ -3446,7 +3449,8 @@ class DeepseekV4CudaDecoderLayer(DeepseekV4DecoderLayerBase):
self.post_attention_layernorm, accepts_mxfp8=False, hc=self.ffn_hc
)
return (
get_parallel().tp_size == self.self_attn.attn_tp_size == 4
get_parallel().tp_size == self.self_attn.attn_tp_size
and mhc.fused_mhc_tp_ok(self.self_attn.attn_tp_size)
and boundary.norm_fusable
and mhc.can_fuse_post(self.hc_cfg)
)
@@ -4870,7 +4874,7 @@ class DeepseekV4ForCausalLM(nn.Module):
# The only deployments whose fused-collective gates can fire; registration
# must happen eagerly here, before graph capture.
if get_platform().is_blackwell and get_parallel().tp_size == 4:
if get_platform().is_blackwell and mhc.fused_mhc_tp_ok(get_parallel().tp_size):
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
register_fused_all_reduce_comm,
)
@@ -45,6 +45,14 @@ _is_hip = is_hip()
_is_npu = is_npu()
_is_xpu = is_xpu()
# The fused all-reduce epilogues have world-size-templated kernels for TP2 and TP4.
_FUSED_MHC_TP_SIZES = (2, 4)
def fused_mhc_tp_ok(tp_size: int) -> bool:
return tp_size in _FUSED_MHC_TP_SIZES
# ---------------------------------------------------------------------------
# What a layer hands this module
# ---------------------------------------------------------------------------