[Perf] DeepSeek-V4.1: Run the small-batch decode fusions at TP2 (#43838)
Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com> Co-authored-by: nbommi19 <227267412+nbommi19@users.noreply.github.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
weireweire
nbommi19
Po-Han Huang
Claude Opus 5.5
Po-Han Huang
parent
fdd53418a0
commit
72d886efb8
@@ -4119,7 +4119,7 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
|
||||
return swapab_attention(
|
||||
q[..., :16, :],
|
||||
q[..., : layer.tp_q_head_num, :],
|
||||
swa_k_cache,
|
||||
swa_page_indices,
|
||||
swa_topk_lengths,
|
||||
|
||||
@@ -31,7 +31,10 @@ from sglang.kernels.ops.attention.dsv4 import (
|
||||
fused_rope_inplace,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.wo_a import MAX_M as _FUSED_WO_A_MAX_TOKENS
|
||||
from sglang.kernels.ops.attention.dsv4.wo_a import fused_rope_wo_a_bf16
|
||||
from sglang.kernels.ops.attention.dsv4.wo_a import (
|
||||
can_use_fused_wo_a,
|
||||
fused_rope_wo_a_bf16,
|
||||
)
|
||||
from sglang.kernels.ops.attention.flash_mla_sm120 import SM120_DECODE_MAX_TOKENS
|
||||
from sglang.kernels.ops.gemm.dsv4_wo_a import (
|
||||
wo_a_bf16_gemv,
|
||||
@@ -1365,7 +1368,7 @@ class MQALayer(MqaAttentionBase):
|
||||
and not self.use_npu_arch35_mxfp8_wo_a
|
||||
and self.wo_a.weight.dtype == torch.bfloat16
|
||||
and self.wo_a.weight.shape == (self.n_local_groups * self.o_lora_rank, 4096)
|
||||
and (self.n_local_groups, self.o_lora_rank) == (2, 1024)
|
||||
and can_use_fused_wo_a(self.n_local_groups, self.o_lora_rank)
|
||||
)
|
||||
if _is_hip:
|
||||
_hip.init_mqa_layer(self, quant_config)
|
||||
@@ -3324,7 +3327,7 @@ class DeepseekV4DecoderLayerBase(nn.Module):
|
||||
def _can_fuse_ffn_mhc(self) -> bool:
|
||||
"""The MoE counterpart of `_can_fuse_attn_mhc`."""
|
||||
return (
|
||||
self.mlp.tp_size == 4
|
||||
mhc.fused_mhc_tp_ok(self.mlp.tp_size)
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not self.mlp._shared_expert_tp1
|
||||
and not should_skip_post_experts_all_reduce(is_tp_path=True)
|
||||
@@ -3446,7 +3449,8 @@ class DeepseekV4CudaDecoderLayer(DeepseekV4DecoderLayerBase):
|
||||
self.post_attention_layernorm, accepts_mxfp8=False, hc=self.ffn_hc
|
||||
)
|
||||
return (
|
||||
get_parallel().tp_size == self.self_attn.attn_tp_size == 4
|
||||
get_parallel().tp_size == self.self_attn.attn_tp_size
|
||||
and mhc.fused_mhc_tp_ok(self.self_attn.attn_tp_size)
|
||||
and boundary.norm_fusable
|
||||
and mhc.can_fuse_post(self.hc_cfg)
|
||||
)
|
||||
@@ -4870,7 +4874,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
|
||||
# The only deployments whose fused-collective gates can fire; registration
|
||||
# must happen eagerly here, before graph capture.
|
||||
if get_platform().is_blackwell and get_parallel().tp_size == 4:
|
||||
if get_platform().is_blackwell and mhc.fused_mhc_tp_ok(get_parallel().tp_size):
|
||||
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
|
||||
register_fused_all_reduce_comm,
|
||||
)
|
||||
|
||||
@@ -45,6 +45,14 @@ _is_hip = is_hip()
|
||||
_is_npu = is_npu()
|
||||
_is_xpu = is_xpu()
|
||||
|
||||
# The fused all-reduce epilogues have world-size-templated kernels for TP2 and TP4.
|
||||
_FUSED_MHC_TP_SIZES = (2, 4)
|
||||
|
||||
|
||||
def fused_mhc_tp_ok(tp_size: int) -> bool:
|
||||
return tp_size in _FUSED_MHC_TP_SIZES
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# What a layer hands this module
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user