[Dsv4.1] Mega-mhc integration (#42808)

Co-authored-by: Yuwei An <yuwei.an@MacBook-Pro-yuweian.local>
Co-authored-by: kpham-sgl <khoa.pham@radixark.ai>
This commit is contained in:
Yuwei An
2026-10-10 17:25:17 -07:00
committed by GitHub
co-authored by Yuwei An kpham-sgl
parent 7187bb593e
commit d32d8fa0d1
4 changed files with 137 additions and 9 deletions
@@ -0,0 +1,68 @@
"""SGLang DSv4.1 prefill integration of Mega mHC.
Credit: DeepSeek's DeepGEMM project provides the fused Mega mHC kernel.
https://github.com/deepseek-ai/DeepGEMM
"""
import torch
from sglang.srt.utils import is_cuda
if is_cuda():
import deep_gemm
def mhc_mega_boundary(
x,
residual,
pre,
post,
comb,
fn,
scale,
base,
weight,
rms_eps,
hc_eps,
norm_eps,
sinkhorn_iters,
):
"""Update the residual, collapse with carried pre, and compute next stats.
The collapse uses ``pre`` from the preceding sublayer, not the new pre
produced from the updated residual. Preserve this DSv4.1 shifted ordering.
Used by the opt-in SM10x prefill path. Before CUDA graph capture, warm up
on the capture stream to initialize DeepGEMM's per-stream barriers.
"""
rows, hidden = x.shape
hc = residual.shape[1]
updated = torch.empty_like(residual)
normalized = torch.empty_like(x)
next_pre = torch.empty((rows, hc, 1), device=x.device, dtype=torch.float32)
next_post = torch.empty_like(next_pre)
next_comb = torch.empty_like(comb)
deep_gemm.mega_mhc(
x=x,
residual=residual,
shifted_prev_mix=pre.unsqueeze(-1),
post_mix=post.unsqueeze(-1),
comb_res_mix=comb,
fn=fn,
mix_scales=scale,
mix_bases=base,
hc_mult=hc,
hc_norm_eps=rms_eps,
hc_pre_eps=hc_eps,
hc_post_scale=2.0,
sinkhorn_eps=hc_eps,
num_sinkhorn_iters=sinkhorn_iters,
rmsnorm_weight=weight,
rmsnorm_eps=norm_eps,
rmsnorm_scale=1.0,
new_residual=updated,
new_prev_mix=next_pre,
new_post_mix=next_post,
new_comb_res_mix=next_comb,
y_bf16=normalized,
)
return updated, normalized, (next_pre.squeeze(-1), next_post.squeeze(-1), next_comb)
+1
View File
@@ -1597,6 +1597,7 @@ class Envs:
# DeepSeek-V4.1 engram host table: keep the tables in host memory (layout
# below) and gather rows from the GPU instead of sharding them over HBM.
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE = EnvBool(False)
SGLANG_OPT_DSV41_MEGA_MHC_PREFILL = EnvBool(False)
# "shared" is one buffer for the whole TP group, mapped by every rank, with no
# lookup all-reduce (the ranks must share a PID namespace); "per_rank" is one
# anonymous mapping per rank holding only its rows, gathered with the
+16 -2
View File
@@ -2842,12 +2842,14 @@ class DeepseekV4DecoderLayer(nn.Module):
self.local_boundary = mhc.make_boundary(
self.post_attention_layernorm,
accepts_mxfp8=False,
hc=self.ffn_hc,
)
# MOE[i] -> ATTN[i+1]
if self._next_layer is not None:
self.next_boundary = mhc.make_boundary(
self._next_layer.input_layernorm,
accepts_mxfp8=self._next_layer.self_attn.accepts_mxfp8_swizzled_input(),
hc=self._next_layer.attn_hc,
)
def refresh_mhc_norm_weight_cache(self):
@@ -3297,11 +3299,16 @@ class DeepseekV4DecoderLayer(nn.Module):
forward_batch: ForwardBatch,
input_ids_global: torch.Tensor,
seam_open: bool = True,
mega_mhc_seam_open: bool = True,
) -> mhc.HcState:
"""The layer's two hyper-connections, each collapsing with the previous one's
pre-mix. ``seam_open`` is False when the late-layer tail narrows the rows after
this layer, so nothing precomputed for the next one would still describe it."""
pre-mix. ``seam_open`` preserves the existing tail-mode boundary policy;
``mega_mhc_seam_open`` disables Mega mHC for both sublayers in the layer
immediately before row selection."""
self._init_boundaries()
mega_mhc_prefill = mhc.can_use_mega_mhc_prefill(
self.hc_cfg, seam_open=mega_mhc_seam_open
)
stats_stream = None
if mhc.use_stats_stream(self.hc_cfg, forward_batch, state.residual):
stats_stream = self.hc_stats_stream
@@ -3312,6 +3319,7 @@ class DeepseekV4DecoderLayer(nn.Module):
quantized = [] if self.self_attn.accepts_mxfp8_swizzled_input() else None
mhc.fork_stats_stream(stats_stream)
x = mhc.combine(self.attn_hc, state, quantized)
precomputed = state.stats
state.release()
del state
world_size = self.self_attn.attn_tp_size
@@ -3334,6 +3342,8 @@ class DeepseekV4DecoderLayer(nn.Module):
stats_stream=stats_stream,
next=self.local_boundary,
world_size=world_size,
precomputed=precomputed,
mega_mhc=mega_mhc_prefill,
)
def run_ffn_hc(state: mhc.HcState) -> mhc.HcState:
@@ -3341,6 +3351,7 @@ class DeepseekV4DecoderLayer(nn.Module):
residual = state.residual
mhc.fork_stats_stream(stats_stream)
x = mhc.combine(self.ffn_hc, state)
precomputed = state.stats
state.release()
del state
nxt = self.next_boundary if seam_open else None
@@ -3362,6 +3373,8 @@ class DeepseekV4DecoderLayer(nn.Module):
stats_stream=stats_stream,
next=nxt,
world_size=self.mlp.tp_size,
precomputed=precomputed,
mega_next=self.next_boundary if mega_mhc_prefill else None,
)
return run_ffn_hc(run_attn_hc(state))
@@ -4117,6 +4130,7 @@ class DeepseekV4Model(nn.Module):
forward_batch=forward_batch,
input_ids_global=input_ids_global,
seam_open=tail is None,
mega_mhc_seam_open=i + 1 != self.late_layer_start,
)
state = state.materialized(self.layers[self.end_layer - 1].hc_cfg)
if saved_full is not None:
+52 -7
View File
@@ -28,6 +28,7 @@ import msgspec
import torch
import torch.nn.functional as F
from sglang.kernels.ops.layernorm.mhc_mega import mhc_mega_boundary
from sglang.kernels.ops.layernorm.mhc_post_split_h import mhc_post_split_h
from sglang.srt.environ import envs
from sglang.srt.hardware_backend.npu.utils import is_npu_arch35
@@ -129,6 +130,7 @@ class HcState(msgspec.Struct):
streams: Union[torch.Tensor, HcPending, None]
pre: Optional[torch.Tensor] = None
input: Optional[HcPreOutput] = None
stats: Optional[HcTriplet] = None
def release(self) -> None:
"""Empty the state at its last reader, the combine; the residual the post
@@ -138,6 +140,7 @@ class HcState(msgspec.Struct):
self.streams = None
self.pre = None
self.input = None
self.stats = None
@property
def residual(self) -> torch.Tensor:
@@ -179,6 +182,7 @@ class HcNextBoundary(NamedTuple):
norm: RMSNorm
accepts_mxfp8: bool
norm_fusable: bool
hc: Optional[HcSubLayer] = None
def is_deferred_finalize(routed) -> bool:
@@ -190,7 +194,9 @@ def is_deferred_finalize(routed) -> bool:
return isinstance(routed, FlashInferTrtllmDeferredFinalizeOutput)
def make_boundary(norm: RMSNorm, *, accepts_mxfp8: bool) -> HcNextBoundary:
def make_boundary(
norm: RMSNorm, *, accepts_mxfp8: bool, hc: Optional[HcSubLayer] = None
) -> HcNextBoundary:
"""Can this hyper-connection's norm fold into the previous post, and does its
sublayer take a pre-quantized input."""
return HcNextBoundary(
@@ -204,6 +210,7 @@ def make_boundary(norm: RMSNorm, *, accepts_mxfp8: bool) -> HcNextBoundary:
and norm.weight.shape == (5120,)
and norm.weight.is_contiguous()
),
hc=hc,
)
@@ -550,27 +557,59 @@ def _compute_triplet(
hc: HcSubLayer,
residual: torch.Tensor,
stats_stream: Optional[torch.cuda.Stream],
precomputed: Optional[HcTriplet] = None,
) -> HcTriplet:
"""Issue the triplet (on the side stream, beside the sublayer) and join before
the post reads it."""
if precomputed is not None:
return precomputed
coefficients = mix_stats(hc, residual, stats_stream)
if stats_stream is not None:
torch.cuda.current_stream().wait_stream(stats_stream)
return coefficients
def can_use_mega_mhc_prefill(cfg: HcConfig, *, seam_open: bool = True) -> bool:
return (
envs.SGLANG_OPT_DSV41_MEGA_MHC_PREFILL.get()
and get_platform().is_sm100
and not _is_hip
and not cfg.cp_prefill
and seam_open
)
def _post_fusion(
hc: HcSubLayer,
y: torch.Tensor,
residual: torch.Tensor,
coefficients: HcTriplet,
next: Optional[HcNextBoundary],
mega_next: Optional[HcNextBoundary] = None,
) -> HcState:
"""Step 4 without a collective: the wide-tile kernel folds the next combine +
norm in where it serves this seam, otherwise the pure post runs and the next
combine computes itself."""
cfg = hc.cfg
pre, post_mix, comb = coefficients
if mega_next is not None and mega_next.norm_fusable and mega_next.hc is not None:
nxt = mega_next.hc
updated, normalized, stats = mhc_mega_boundary(
y,
residual,
pre,
post_mix,
comb,
nxt.fn,
nxt.scale,
nxt.base,
nxt.norm.weight,
nxt.cfg.rms_eps,
nxt.cfg.eps,
nxt.norm.variance_epsilon,
nxt.cfg.sinkhorn_iters,
)
return HcState(updated, pre, HcNormed(normalized), stats)
if (
next is not None
and next.norm_fusable
@@ -607,6 +646,8 @@ def run_attn_post(
stats_stream: Optional[torch.cuda.Stream],
next: Optional[HcNextBoundary],
world_size: int,
precomputed: Optional[HcTriplet] = None,
mega_mhc: bool = False,
) -> HcState:
"""The attention post. An `AttnOutput` rides the collective kernel (which also
folds ``next``'s norm); the attention may decline the handover even when asked,
@@ -617,7 +658,7 @@ def run_attn_post(
)
assert next is not None
pre, post_mix, comb = _compute_triplet(hc, residual, stats_stream)
pre, post_mix, comb = _compute_triplet(hc, residual, stats_stream, precomputed)
_, updated, normalized = all_reduce_mhc_post_combine_norm(
out.partial,
residual,
@@ -629,8 +670,10 @@ def run_attn_post(
world_size=world_size,
)
return HcState(updated, pre, HcNormed(normalized))
coefficients = _compute_triplet(hc, residual, stats_stream)
return _post_fusion(hc, out, residual, coefficients, next)
coefficients = _compute_triplet(hc, residual, stats_stream, precomputed)
return _post_fusion(
hc, out, residual, coefficients, next, next if mega_mhc else None
)
def run_moe_post(
@@ -641,6 +684,8 @@ def run_moe_post(
stats_stream: Optional[torch.cuda.Stream],
next: Optional[HcNextBoundary],
world_size: int,
precomputed: Optional[HcTriplet] = None,
mega_next: Optional[HcNextBoundary] = None,
) -> HcState:
"""The MoE post. A deferred finalize the push plane can carry rides the
collective kernel (finalize + shared add + all-reduce, quantizing ``next``'s
@@ -657,7 +702,7 @@ def run_moe_post(
# [T x top_k (padded)] view and overstates the plane load by ~6x.
and can_fuse_all_reduce(out.routed.expert_weights.shape[0], hc.cfg.hidden)
):
pre, post_mix, comb = _compute_triplet(hc, residual, stats_stream)
pre, post_mix, comb = _compute_triplet(hc, residual, stats_stream, precomputed)
args = (
out.routed.gemm2_out,
out.routed.expanded_idx_to_permuted_idx,
@@ -711,5 +756,5 @@ def run_moe_post(
and should_add_replicated_moe_output()
):
out += pieces.shared
coefficients = _compute_triplet(hc, residual, stats_stream)
return _post_fusion(hc, out, residual, coefficients, next)
coefficients = _compute_triplet(hc, residual, stats_stream, precomputed)
return _post_fusion(hc, out, residual, coefficients, next, mega_next)