[Perf] dsv4: Combine the final mHC streams from bf16 without an fp32 copy (#45940)

This commit is contained in:
Liangsheng Yin
2026-10-22 19:43:46 -07:00
committed by GitHub
parent 363e6c6b4d
commit 2da6c0c237
2 changed files with 3 additions and 1 deletions
@@ -2718,6 +2718,7 @@ def mhc_combine(
x_flat: torch.Tensor, pre: torch.Tensor, hc: int, out_dtype: torch.dtype
) -> torch.Tensor:
"""Fused y[m, h] = sum_k pre[m, k] * x_flat[m, k*H + h]."""
assert x_flat.stride(1) == 1
m = x_flat.shape[0]
h = x_flat.shape[1] // hc
y = torch.empty((m, h), dtype=out_dtype, device=x_flat.device)
+2 -1
View File
@@ -4464,8 +4464,9 @@ class DeepseekV4Model(nn.Module):
if self.hc_pre_from_prev_sublayer:
from sglang.kernels.ops.layernorm.mhc import mhc_combine
# The kernel widens each element to fp32 as it loads it.
hidden_states = mhc_combine(
pre_hc_head.float(), last_pre, self.hc_mult, hidden_states.dtype
pre_hc_head, last_pre, self.hc_mult, hidden_states.dtype
)
else:
hidden_states = self.hc_head(