[Perf] dsv4: Combine the final mHC streams from bf16 without an fp32 copy (#45940)
This commit is contained in:
@@ -2718,6 +2718,7 @@ def mhc_combine(
|
||||
x_flat: torch.Tensor, pre: torch.Tensor, hc: int, out_dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
"""Fused y[m, h] = sum_k pre[m, k] * x_flat[m, k*H + h]."""
|
||||
assert x_flat.stride(1) == 1
|
||||
m = x_flat.shape[0]
|
||||
h = x_flat.shape[1] // hc
|
||||
y = torch.empty((m, h), dtype=out_dtype, device=x_flat.device)
|
||||
|
||||
@@ -4464,8 +4464,9 @@ class DeepseekV4Model(nn.Module):
|
||||
if self.hc_pre_from_prev_sublayer:
|
||||
from sglang.kernels.ops.layernorm.mhc import mhc_combine
|
||||
|
||||
# The kernel widens each element to fp32 as it loads it.
|
||||
hidden_states = mhc_combine(
|
||||
pre_hc_head.float(), last_pre, self.hc_mult, hidden_states.dtype
|
||||
pre_hc_head, last_pre, self.hc_mult, hidden_states.dtype
|
||||
)
|
||||
else:
|
||||
hidden_states = self.hc_head(
|
||||
|
||||
Reference in New Issue
Block a user