From 2da6c0c237324eefea6420e015beb1e6858cb80c Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 22 Oct 2026 19:43:46 -0700 Subject: [PATCH] [Perf] dsv4: Combine the final mHC streams from bf16 without an fp32 copy (#45940) --- python/sglang/kernels/ops/layernorm/mhc.py | 1 + python/sglang/srt/models/deepseek_v4.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python/sglang/kernels/ops/layernorm/mhc.py b/python/sglang/kernels/ops/layernorm/mhc.py index d7f69f4068..e1f29ab61b 100644 --- a/python/sglang/kernels/ops/layernorm/mhc.py +++ b/python/sglang/kernels/ops/layernorm/mhc.py @@ -2718,6 +2718,7 @@ def mhc_combine( x_flat: torch.Tensor, pre: torch.Tensor, hc: int, out_dtype: torch.dtype ) -> torch.Tensor: """Fused y[m, h] = sum_k pre[m, k] * x_flat[m, k*H + h].""" + assert x_flat.stride(1) == 1 m = x_flat.shape[0] h = x_flat.shape[1] // hc y = torch.empty((m, h), dtype=out_dtype, device=x_flat.device) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 2142b5e2f5..f880594602 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -4464,8 +4464,9 @@ class DeepseekV4Model(nn.Module): if self.hc_pre_from_prev_sublayer: from sglang.kernels.ops.layernorm.mhc import mhc_combine + # The kernel widens each element to fp32 as it loads it. hidden_states = mhc_combine( - pre_hc_head.float(), last_pre, self.hc_mult, hidden_states.dtype + pre_hc_head, last_pre, self.hc_mult, hidden_states.dtype ) else: hidden_states = self.hc_head(