[DeepSeek-V4.1] Unify mHC into one state machine and drop medium-batch fusion variants (#42245)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: BBuf <1182563586@qq.com>
This commit is contained in:
DarkSharpness
2026-10-06 21:01:42 +08:00
committed by GitHub
co-authored by Claude Fable 5 BBuf
parent 2a5f45d3d4
commit 730f1f3e5b
19 changed files with 1359 additions and 1503 deletions
@@ -510,7 +510,7 @@ struct MoeFinalizeAllReduceKernel {
}
/// Finalize + all-reduce + HC=4 post; original reduced output is retained.
static void run_mhc(
static void run_mhc_post(
CommunicatorRef ref,
TensorView out,
TensorView gemm2_out,
@@ -538,41 +538,7 @@ struct MoeFinalizeAllReduceKernel {
comb);
}
// Keep the original RMSNorm as a separate kernel, while reusing the
// BF16-rounded post values for the next sublayer's pre-combine.
static void run_mhc_combine(
CommunicatorRef ref,
TensorView out,
TensorView gemm2_out,
TensorView permuted_idx,
TensorView expert_weights,
std::optional<TensorView> shared_output,
TensorView mhc_out,
TensorView residual,
TensorView post,
TensorView comb,
TensorView pre,
TensorView combined) {
static_assert(kMhc && kCollapse && !kQuant);
run_impl(
ref,
out,
gemm2_out,
permuted_idx,
expert_weights,
shared_output,
std::nullopt,
0.0,
false,
mhc_out,
residual,
post,
comb,
pre,
combined);
}
static void run_mhc_norm(
static void run_mhc_post_combine_norm(
CommunicatorRef ref,
TensorView out,
TensorView gemm2_out,
@@ -606,7 +572,7 @@ struct MoeFinalizeAllReduceKernel {
normalized);
}
static void run_mhc_quant(
static void run_mhc_post_combine_norm_quant(
CommunicatorRef ref,
TensorView out,
TensorView gemm2_out,
@@ -707,7 +673,7 @@ struct MoeFinalizeAllReduceKernel {
}
const auto num_tokens = static_cast<uint32_t>(T.unwrap());
if constexpr (kQuant) {
CHECK_HOST(num_tokens <= 8);
CHECK_HOST(num_tokens <= 128) << "the quant epilogue writes one 128-row scale tile";
CHECK_HOST(norm_weight.has_value());
TensorMatcher({T, kHiddenDim}).with_dtype<fp8_e4m3_t>().with_device<kDLCUDA>(device).verify(quantized.value());
TensorMatcher({(kHiddenDim / 32) * 128})
@@ -34,18 +34,21 @@ def _jit_mhc_module(world_size, top_k, cluster_size, weight_dtype):
True,
)
return load_jit(
"moe_finalize_all_reduce_mhc",
"moe_finalize_all_reduce_mhc_post",
*args,
cuda_files=["distributed/all_reduce_fusion.cuh"],
cuda_wrappers=[
("run", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc"),
("run_norm", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_norm"),
("run", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_post"),
(
"run_norm",
f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_post_combine_norm",
),
],
)
@register_custom_op(mutates_args=["out", "mhc_out"])
def _moe_finalize_all_reduce_mhc_op(
def _moe_finalize_all_reduce_mhc_post_op(
world_size: int,
top_k: int,
cluster_size: int,
@@ -75,7 +78,7 @@ def _moe_finalize_all_reduce_mhc_op(
)
def moe_finalize_all_reduce_mhc(
def moe_finalize_all_reduce_mhc_post(
gemm2: torch.Tensor,
idx: torch.Tensor,
weights: torch.Tensor,
@@ -99,7 +102,7 @@ def moe_finalize_all_reduce_mhc(
)
mhc_out = torch.empty_like(residual)
if weights.shape[0]:
_moe_finalize_all_reduce_mhc_op(
_moe_finalize_all_reduce_mhc_post_op(
world_size,
top_k,
cluster_size or default_cluster_size(_MHC_HIDDEN_DIM),
@@ -117,7 +120,7 @@ def moe_finalize_all_reduce_mhc(
@register_custom_op(mutates_args=["out", "mhc_out", "normalized"])
def _moe_finalize_all_reduce_mhc_norm_op(
def _moe_finalize_all_reduce_mhc_post_combine_norm_op(
world_size: int,
top_k: int,
cluster_size: int,
@@ -155,7 +158,7 @@ def _moe_finalize_all_reduce_mhc_norm_op(
)
def moe_finalize_all_reduce_mhc_norm(
def moe_finalize_all_reduce_mhc_post_combine_norm(
gemm2: torch.Tensor,
idx: torch.Tensor,
weights: torch.Tensor,
@@ -171,7 +174,7 @@ def moe_finalize_all_reduce_mhc_norm(
world_size: int,
cluster_size: Optional[int] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
""":func:`moe_finalize_all_reduce_mhc` plus the HC=4 pre-collapse
""":func:`moe_finalize_all_reduce_mhc_post` plus the HC=4 pre-collapse
(``pre`` ``[T, 4]`` fp32) and RMSNorm of the collapsed row. Returns
``(reduced, mhc_out, normalized [T, hidden])``.
"""
@@ -181,7 +184,7 @@ def moe_finalize_all_reduce_mhc_norm(
mhc_out = torch.empty_like(residual)
normalized = torch.empty_like(out)
if weights.shape[0]:
_moe_finalize_all_reduce_mhc_norm_op(
_moe_finalize_all_reduce_mhc_post_combine_norm_op(
world_size,
top_k,
cluster_size or default_cluster_size(_MHC_HIDDEN_DIM),
@@ -210,7 +213,7 @@ def _identity_routing(rows, device):
)
def all_reduce_mhc_norm(
def all_reduce_mhc_post_combine_norm(
x: torch.Tensor,
residual: torch.Tensor,
post: torch.Tensor,
@@ -224,7 +227,7 @@ def all_reduce_mhc_norm(
"""Plain all-reduce of ``x`` with the mHC + norm epilogue: the finalize
kernel driven with identity routing (top_k = 1, unit weights)."""
idx, weights = _identity_routing(x.shape[0], x.device)
return moe_finalize_all_reduce_mhc_norm(
return moe_finalize_all_reduce_mhc_post_combine_norm(
x,
idx,
weights,
@@ -254,17 +257,22 @@ def _jit_mhc_quant_module(world_size, top_k, cluster_size, weight_dtype):
True,
)
return load_jit(
"moe_finalize_all_reduce_mhc_quant",
"moe_finalize_all_reduce_mhc_post_combine_norm_quant",
*args,
cuda_files=["distributed/all_reduce_fusion.cuh"],
cuda_wrappers=[("run", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_quant")],
cuda_wrappers=[
(
"run",
f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_post_combine_norm_quant",
)
],
)
@register_custom_op(
mutates_args=["out", "mhc_out", "normalized", "quantized", "scales"]
)
def _moe_finalize_all_reduce_mhc_quant_op(
def _moe_finalize_all_reduce_mhc_post_combine_norm_quant_op(
world_size: int,
top_k: int,
cluster_size: int,
@@ -306,7 +314,7 @@ def _moe_finalize_all_reduce_mhc_quant_op(
)
def moe_finalize_all_reduce_mhc_quant(
def moe_finalize_all_reduce_mhc_post_combine_norm_quant(
gemm2: torch.Tensor,
idx: torch.Tensor,
weights: torch.Tensor,
@@ -322,12 +330,14 @@ def moe_finalize_all_reduce_mhc_quant(
world_size: int,
cluster_size: Optional[int] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
""":func:`moe_finalize_all_reduce_mhc_norm` plus fp8 e4m3 quantization of
the normalized row with ue8m0 group scales (rows <= 8). Returns
""":func:`moe_finalize_all_reduce_mhc_post_combine_norm` plus fp8 e4m3 quantization of
the normalized row with ue8m0 group scales (rows <= 128: the scale swizzle
addresses a single 128-row tile). Returns
``(reduced, mhc_out, normalized, quantized, scales)``.
"""
rows = weights.shape[0]
assert 0 < rows <= 8
# One 128-row scale tile; larger needs a (token/128) block offset in the kernel.
assert 0 < rows <= 128
out = torch.empty(
(rows, _MHC_HIDDEN_DIM), dtype=torch.bfloat16, device=gemm2.device
)
@@ -338,7 +348,7 @@ def moe_finalize_all_reduce_mhc_quant(
scales = torch.empty(
(_MHC_HIDDEN_DIM // 32) * 128, device=gemm2.device, dtype=torch.uint8
)
_moe_finalize_all_reduce_mhc_quant_op(
_moe_finalize_all_reduce_mhc_post_combine_norm_quant_op(
world_size,
top_k,
cluster_size or default_cluster_size(_MHC_HIDDEN_DIM),
@@ -1,121 +0,0 @@
"""Finalize/all-reduce + post + combine, retaining standalone RMSNorm."""
from typing import Optional
import torch
from sglang.kernels.jit.utils import (
cache_once,
is_arch_support_pdl,
load_jit,
make_cpp_args,
)
from sglang.kernels.ops.communication.all_reduce_fusion import (
default_cluster_size,
get_registered_comm,
)
from sglang.srt.utils.custom_op import register_custom_op
@cache_once
def _module(world_size, top_k, cluster_size, weight_dtype):
args = make_cpp_args(
world_size,
5120,
top_k,
cluster_size,
is_arch_support_pdl(),
weight_dtype,
True,
False,
True,
)
return load_jit(
"moe_finalize_all_reduce_mhc_combine",
*args,
cuda_files=["distributed/all_reduce_fusion.cuh"],
cuda_wrappers=[("run", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_combine")],
)
@register_custom_op(mutates_args=["out", "mhc_out", "combined"])
def _moe_finalize_all_reduce_mhc_combine(
world_size: int,
top_k: int,
cluster_size: int,
out: torch.Tensor,
mhc_out: torch.Tensor,
combined: torch.Tensor,
gemm2: torch.Tensor,
idx: torch.Tensor,
weights: torch.Tensor,
shared: Optional[torch.Tensor],
residual: torch.Tensor,
post: torch.Tensor,
comb: torch.Tensor,
pre: torch.Tensor,
) -> None:
comm = get_registered_comm(world_size)
assert comm is not None
_module(world_size, top_k, cluster_size, weights.dtype).run(
comm,
out,
gemm2,
idx,
weights,
shared,
mhc_out,
residual,
post,
comb,
pre,
combined,
)
def moe_finalize_all_reduce_mhc_combine(
gemm2,
idx,
weights,
top_k,
shared,
residual,
post,
comb,
pre,
*,
world_size,
cluster_size=None,
):
out = torch.empty(
(weights.shape[0], 5120), dtype=torch.bfloat16, device=gemm2.device
)
mhc_out = torch.empty_like(residual)
combined = torch.empty_like(out)
if weights.shape[0]:
_moe_finalize_all_reduce_mhc_combine(
world_size,
top_k,
cluster_size or default_cluster_size(5120),
out,
mhc_out,
combined,
gemm2,
idx,
weights,
shared,
residual,
post,
comb,
pre,
)
return out, mhc_out, combined
def all_reduce_mhc_combine(x, residual, post, comb, pre, *, world_size):
from sglang.kernels.ops.communication.all_reduce_mhc import _identity_routing
idx, weights = _identity_routing(x.shape[0], x.device)
return moe_finalize_all_reduce_mhc_combine(
x, idx, weights, 1, None, residual, post, comb, pre, world_size=world_size
)
@@ -55,9 +55,12 @@ def _hc_combine_norm_prefill(
def hc_combine_norm(
x: torch.Tensor, pre: torch.Tensor, weight: torch.Tensor, eps: float
) -> torch.Tensor:
"""Fuse four-stream combine and RMSNorm for BF16 batches of width 5120."""
"""Fuse four-stream combine and RMSNorm for BF16 batches of width 5120.
Every row count takes the same per-row reduction in the same order, so the
result is batch-invariant across the grid-split variants below."""
m = x.shape[0]
assert (0 < m <= 96 or 4096 <= m <= 65536) and x.shape == (m, 20480)
assert x.shape == (m, 20480)
assert pre.shape == (m, 4) and pre.stride(1) == 1
assert weight.shape == (5120,) and weight.is_contiguous()
assert x.dtype == weight.dtype == torch.bfloat16 and x.stride(1) == 1
@@ -133,7 +136,7 @@ def hc_combine_norm_mxfp8(
):
"""Four-stream combine + RMSNorm returning ``(y_bf16, y_q, y_sf)``."""
m = x.shape[0]
assert 0 < m <= 8, "the fused MXFP8 epilogue only supports small decode/verify"
assert m <= 128, "the MXFP8 scale swizzle addresses a single 128-row tile"
k = x.shape[1] // 4
y = torch.empty((m, k), dtype=x.dtype, device=x.device)
q, s = _alloc(m, k, x.device)
@@ -1,85 +0,0 @@
"""HC=4 post-mix and pre-combine with the original BF16 intermediates."""
import torch
import triton
import triton.language as tl
@triton.jit
def _mhc_post_combine(X, R, P, C, A, RO, Y, H: tl.constexpr, B: tl.constexpr):
row = tl.program_id(0)
h = tl.program_id(1) * B + tl.arange(0, B)
mask = h < H
x = tl.load(X + row * H + h, mask, 0).to(tl.float32)
r0 = tl.load(R + (row * 4 + 0) * H + h, mask, 0).to(tl.float32)
r1 = tl.load(R + (row * 4 + 1) * H + h, mask, 0).to(tl.float32)
r2 = tl.load(R + (row * 4 + 2) * H + h, mask, 0).to(tl.float32)
r3 = tl.load(R + (row * 4 + 3) * H + h, mask, 0).to(tl.float32)
collapsed = tl.full((B,), 0, tl.float32)
for i in tl.static_range(4):
post = tl.load(P + row * 4 + i)
c0 = tl.load(C + row * 16 + i)
c1 = tl.load(C + row * 16 + 4 + i)
c2 = tl.load(C + row * 16 + 8 + i)
c3 = tl.load(C + row * 16 + 12 + i)
pre = tl.load(A + row * 4 + i)
# Match mhc_post_split_h's contraction order and BF16 store.
mixed = tl.fma(post, x, c0 * r0)
mixed = tl.fma(c1, r1, mixed)
mixed = tl.fma(c2, r2, mixed)
mixed = tl.fma(c3, r3, mixed)
rounded = mixed.to(tl.bfloat16)
tl.store(RO + (row * 4 + i) * H + h, rounded, mask)
# Match hc_combine's sequential FP32 accumulation, then BF16 store.
collapsed = tl.fma(pre, rounded.to(tl.float32), collapsed)
tl.store(Y + row * H + h, collapsed, mask)
def mhc_post_combine(x, residual, post, comb, pre):
"""Return (updated HC streams, collapsed BF16 input) out of place."""
assert x.dtype == residual.dtype == torch.bfloat16
assert post.dtype == comb.dtype == pre.dtype == torch.float32
assert x.ndim == 2 and x.shape[1] == 5120
assert residual.shape == (x.shape[0], 4, x.shape[1])
assert post.shape == pre.shape == (x.shape[0], 4)
assert comb.shape == (x.shape[0], 4, 4)
assert all(t.is_contiguous() for t in (x, residual, post, comb, pre))
updated = torch.empty_like(residual)
combined = torch.empty_like(x)
if x.shape[0]:
block = 1024 if x.shape[0] <= 192 or x.shape[0] >= 4096 else 512
_mhc_post_combine[(x.shape[0], triton.cdiv(x.shape[1], block))](
x,
residual,
post,
comb,
pre,
updated,
combined,
H=x.shape[1],
B=block,
num_warps=4,
enable_fp_fusion=False,
)
return updated, combined
@triton.jit
def _hc_norm_prefill(X, W, Y, EPS: tl.constexpr):
row = tl.program_id(0).to(tl.int64)
h = tl.arange(0, 8192)
value = tl.load(X + row * 5120 + h, h < 5120, 0).to(tl.float32)
inv_rms = tl.rsqrt(tl.sum(value * value, 0) / 5120 + EPS)
weight = tl.load(W + h, h < 5120, 0).to(tl.float32)
tl.store(Y + row * 5120 + h, value * inv_rms * weight, h < 5120)
def hc_norm_prefill(combined, weight, eps):
"""Keep the original fused prefill norm's arithmetic on collapsed input."""
assert 4096 <= combined.shape[0] <= 65536 and combined.shape[1] == 5120
assert weight.shape == (5120,)
assert combined.dtype == weight.dtype == torch.bfloat16
assert combined.is_contiguous() and weight.is_contiguous()
output = torch.empty_like(combined)
_hc_norm_prefill[(combined.shape[0],)](combined, weight, output, eps, num_warps=4)
return output
@@ -1,57 +0,0 @@
"""Scoped handoff of a decoder's HC post operands to deferred MoE finalize."""
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from typing import Callable, Optional
import torch
@dataclass
class MhcPostFusion:
residual: torch.Tensor
post: Optional[torch.Tensor]
comb: Optional[torch.Tensor]
stats_stream: Optional[torch.cuda.Stream]
output: Optional[torch.Tensor] = None
pre: Optional[torch.Tensor] = None
norm_weight: Optional[torch.Tensor] = None
norm_eps: float = 0.0
normalized: Optional[torch.Tensor] = None
quantized: Optional[tuple[torch.Tensor, torch.Tensor]] = None
combine_only: bool = False
combined: Optional[torch.Tensor] = None
overlap_only: bool = False
record_stats: Optional[
Callable[[], tuple[torch.Tensor, torch.Tensor, torch.Tensor]]
] = None
def start_stats_before_all_reduce(self):
if self.overlap_only and self.record_stats is not None:
assert self.stats_stream is not None
self.stats_stream.wait_stream(torch.cuda.current_stream())
self.materialize_stats()
def materialize_stats(self):
# Record after the main parent; graph replay must keep the join on the
# main stream.
if self.record_stats is not None:
self.pre, self.post, self.comb = self.record_stats()
self.record_stats = None
_current: ContextVar[Optional[MhcPostFusion]] = ContextVar("moe_mhc_post", default=None)
def current_mhc_post_fusion():
return _current.get()
@contextmanager
def use_mhc_post_fusion(state):
token = _current.set(state)
try:
yield
finally:
_current.reset(token)
@@ -81,7 +81,7 @@ def is_deferred_finalize_enabled() -> bool:
def finalize_flashinfer_trtllm_deferred_output(
deferred_output: FlashInferTrtllmDeferredFinalizeOutput,
shared_output: torch.Tensor,
shared_output: Optional[torch.Tensor],
) -> torch.Tensor:
from sglang.kernels.ops.moe.moe_finalize_fuse_shared import moe_finalize_fuse_shared
@@ -519,82 +519,43 @@ def maybe_fuse_routed_scale_and_shared_add(
return routed
# Fused finalize + shared add + TP all-reduce
_fused_finalize_all_reduce_world_size: Optional[int] = None
_fused_finalize_all_reduce_probed = False
_fused_finalize_all_reduce_comm = None
# Fused finalize + shared add + TP all-reduce, staged through the TP group's own
# CustomAllReduceV2 push plane (tiny batches only; no dedicated workspace).
def _fused_finalize_all_reduce_comm_world_size() -> Optional[int]:
"""Reserve a separate push plane for up to 384 rows of fused MoE output."""
global _fused_finalize_all_reduce_world_size, _fused_finalize_all_reduce_probed
global _fused_finalize_all_reduce_comm
if not _fused_finalize_all_reduce_probed:
# The eager warmup must initialize peer workspaces before capture.
if torch.cuda.is_current_stream_capturing():
return None
_fused_finalize_all_reduce_probed = True
from sglang.kernels.ops.communication import all_reduce_fusion
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
CustomAllReduceV2,
def register_fused_all_reduce_comm() -> None:
"""Hand the TP group's push plane to the fusion kernels, once per process.
Must run before graph capture; unregistered means everything runs unfused."""
from sglang.kernels.ops.communication import all_reduce_fusion
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
CustomAllReduceV2,
)
ca_comm = get_parallel().tp_group.ca_comm
if isinstance(ca_comm, CustomAllReduceV2) and not ca_comm.disabled:
all_reduce_fusion.register_comm(ca_comm.obj)
else:
log_info_on_rank0(
logger,
"Fused MoE finalize: TP group has no "
"CustomAllReduceV2 push plane; keeping the unfused finalize path",
)
ca_comm = get_parallel().tp_group.ca_comm
if isinstance(ca_comm, CustomAllReduceV2) and not ca_comm.disabled:
fused_comm = ca_comm
if ca_comm.world_size == 4:
from sglang.kernels.ops.communication.mp import register_comm_cleanup
fused_comm = CustomAllReduceV2(
ca_comm.group,
ca_comm.device,
max_pull_size=0,
max_pull_blocks=0,
max_push_size=4 * 1024 * 1024,
max_push_blocks=512,
)
register_comm_cleanup(fused_comm)
if fused_comm.disabled:
return None
_fused_finalize_all_reduce_comm = fused_comm
all_reduce_fusion.register_comm(fused_comm.obj)
_fused_finalize_all_reduce_world_size = fused_comm.world_size
else:
log_info_on_rank0(
logger,
"Fused MoE finalize: TP group has no "
"CustomAllReduceV2 push plane; keeping the unfused finalize path",
)
return _fused_finalize_all_reduce_world_size
def can_fuse_all_reduce(num_tokens: int, hidden_dim: int) -> bool:
"""Whether the registered push plane can stage ``[num_tokens, hidden_dim]``
BF16 rows: one slot holds them and each row owns a push phase counter. The
384 ceiling is the bench-validated range of the fused collective family.
def should_use_fuse_finalize_all_reduce(
experts, num_tokens: int, hidden_dim: int
) -> bool:
"""Capability only; the batch-size policy cap lives at the call site. The
kernel never rescales, so the expert weights must carry the routed scaling."""
if not isinstance(experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod):
return False
if experts.quant_method.flashinfer_mxfp4_moe_precision != "default":
return False
if not experts.should_fuse_routed_scaling_factor_in_topk:
return False
if num_tokens <= 0:
return False
if num_tokens > 96:
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
if is_batch_invariant_mode_enabled():
return False
The deferred-finalize handle type carries its own numeric contract (every
producer folds the routed scaling; no consumer rescales)."""
from sglang.kernels.ops.communication import all_reduce_fusion
if not all_reduce_fusion.valid_cluster_sizes(hidden_dim):
if all_reduce_fusion.get_registered_comm(get_parallel().tp_size) is None:
return False
tp_group = get_parallel().tp_group
if _fused_finalize_all_reduce_comm_world_size() != tp_group.world_size:
return False
comm = _fused_finalize_all_reduce_comm
# Each token row owns a push phase counter on the epilogue's plane.
if num_tokens > comm.config.num_push_blocks:
return False
return all_reduce_fusion.fits_push_slot(comm.max_push_size, num_tokens, hidden_dim)
comm = get_parallel().tp_group.ca_comm
return (
num_tokens <= 384
and num_tokens <= comm.config.num_push_blocks
and all_reduce_fusion.fits_push_slot(comm.max_push_size, num_tokens, hidden_dim)
)
@@ -7,13 +7,7 @@ from typing import Optional, Tuple
import torch
from sglang.kernels.ops.communication.all_reduce_mhc_hip import all_reduce_mhc_post
from sglang.srt.layers.moe.mhc_post_fusion import MhcPostFusion
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import (
ALL_REDUCE_MHC_MAX_ROWS,
)
from sglang.srt.models.deepseek_common.utils import _use_aiter
from sglang.srt.runtime_context import get_parallel
if _use_aiter:
from sglang.kernels.ops.moe.rocm_router_gate import rocm_router_max_tokens
@@ -58,27 +52,3 @@ def forward_gate(
if logits_and_partials is not None:
return logits_and_partials
return moe.gate(hidden_states, gemm_output_zero_allocator), None
def fused_all_reduce_mhc(
moe, mhc: Optional[MhcPostFusion], hidden_states: torch.Tensor
) -> bool:
"""Whether the post-experts all-reduce of moe (a DeepseekV2MoE) ran fused with the
mHC post, for the eagerly built 1-8 row mHC states; the decoder then reads mhc.output
and hidden_states only keeps the DP wrapper's tensor contract."""
if not (
mhc is not None
and not moe._shared_expert_tp1
and not mhc.overlap_only
and mhc.post is not None
and 1 <= hidden_states.shape[0] <= ALL_REDUCE_MHC_MAX_ROWS
):
return False
mhc.output = all_reduce_mhc_post(
hidden_states,
mhc.residual,
mhc.post,
mhc.comb,
get_parallel().tp_group.ca_comm,
)
return True
@@ -1,5 +1,5 @@
import logging
from typing import TYPE_CHECKING, Optional, Tuple
from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple
import torch
import triton
@@ -10,7 +10,9 @@ if TYPE_CHECKING:
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.environ import envs
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.moe.mhc_post_fusion import MhcPostFusion, use_mhc_post_fusion
from sglang.srt.layers.moe.utils import should_skip_post_experts_all_reduce
from sglang.srt.models.deepseek_v2 import MoEOutput
from sglang.srt.models.deepseek_v4_mhc import AttnOutput
from sglang.srt.runtime_context import (
get_exec,
get_forward,
@@ -38,6 +40,19 @@ if _is_gfx95_supported:
post_attention_norm,
)
class MhcPostOperands(NamedTuple):
"""What the fused all-reduce + hc_post kernel reads besides the sublayer rows.
The caller hands this to `apply_attention_mhc` / `apply_moe_mhc` directly; the
sublayer only has to be told to leave its reduce alone.
"""
residual: torch.Tensor
post: torch.Tensor
comb: torch.Tensor
# the fused all-reduce + hc_post kernel: DeepSeek-V4.1's hidden size, up to this many rows
MHC_HIDDEN_SIZE = 5120
ALL_REDUCE_MHC_MAX_ROWS = 8
@@ -422,7 +437,7 @@ def apply_mhc_post_pre_boundary(
if not _is_fused_mhc_post_pre_enabled():
return None
from sglang.srt.models.deepseek_v4 import _get_mhc_ops
from sglang.srt.models.deepseek_v4 import get_mhc_ops
post_in = post.unsqueeze(-1) if post.ndim == 2 else post
(
@@ -430,7 +445,7 @@ def apply_mhc_post_pre_boundary(
post_out,
comb_out,
layer_input_out,
) = _get_mhc_ops().mhc_fused_post_pre(
) = get_mhc_ops().mhc_fused_post_pre(
layer_input,
residual,
post_in,
@@ -508,14 +523,12 @@ def _can_fuse_mhc(layer, residual: torch.Tensor, forward_batch) -> bool:
def _make_mhc_fusion(
residual: torch.Tensor, coefficients: "HcCoefficients", comm
) -> Optional[MhcPostFusion]:
) -> Optional[MhcPostOperands]:
# comm is the group's aiter CustomAllreduce, or None when custom all-reduce is off
if comm is None or comm.disabled or not comm.enable_register_for_capturing:
return None
coefficients.materialize()
return MhcPostFusion(
residual, coefficients.post, coefficients.comb, None, pre=coefficients.pre
)
return MhcPostOperands(residual, coefficients.post, coefficients.comb)
def attention_mhc_fusion(layer, residual, coefficients, forward_batch):
@@ -536,21 +549,29 @@ def moe_mhc_fusion(layer, residual, coefficients, forward_batch):
and layer.mlp.tp_size == 4
and not layer.mlp._shared_expert_tp1
and get_moe_a2a_backend().is_none()
# A reduce some later step already owns, or one the combine performed, is
# not ours to fuse into the post.
and not should_skip_post_experts_all_reduce(is_tp_path=True)
):
return None
return _make_mhc_fusion(residual, coefficients, get_parallel().tp_group.ca_comm)
def apply_attention_mhc(x: torch.Tensor, state: MhcPostFusion) -> None:
# a lazily recorded state still needs its stats before the fused kernel reads them
state.materialize_stats()
if state.stats_stream is not None:
torch.cuda.current_stream().wait_stream(state.stats_stream)
state.output = all_reduce_mhc_post(
def apply_attention_mhc(x: torch.Tensor, state: MhcPostOperands) -> torch.Tensor:
"""Attention's all-reduce fused with hc_post; returns the rebuilt streams."""
return all_reduce_mhc_post(
x, state.residual, state.post, state.comb, get_parallel().attn_tp_group.ca_comm
)
def apply_moe_mhc(x: torch.Tensor, state: MhcPostOperands) -> torch.Tensor:
"""The same for the MoE's post-experts reduce, on rows it merged but left
unreduced."""
return all_reduce_mhc_post(
x, state.residual, state.post, state.comb, get_parallel().tp_group.ca_comm
)
def forward_hc_pre_from_prev_fused_boundary(
layer,
positions: torch.Tensor,
@@ -581,19 +602,25 @@ def forward_hc_pre_from_prev_fused_boundary(
)
with layer.self_attn.maybe_use_decode_attn_tp(forward_batch):
mhc = attention_mhc_fusion(layer, residual, attn_coefficients, forward_batch)
with use_mhc_post_fusion(mhc):
x = layer.self_attn(
x=x, positions=positions, forward_batch=forward_batch, x_quant=x_quant
)
x = layer.self_attn(
x=x,
positions=positions,
forward_batch=forward_batch,
x_quant=x_quant,
defer_all_reduce=mhc is not None,
)
if mhc is not None:
residual = mhc.output
# wo_b honors the deferred reduce under attention_mhc_fusion's gates;
# fail loud rather than fall back to an unfused reduce.
assert isinstance(x, AttnOutput)
residual = apply_attention_mhc(x.partial, mhc)
x = None
residual, x, ffn_coefficients = hc_boundary(
layer,
x,
residual,
attn_coefficients.post if mhc is None else None,
attn_coefficients.comb if mhc is None else None,
attn_coefficients.post if x is not None else None,
attn_coefficients.comb if x is not None else None,
attn_coefficients.pre,
layer.hc_ffn_fn,
layer.hc_ffn_scale,
@@ -601,15 +628,22 @@ def forward_hc_pre_from_prev_fused_boundary(
)
x = _gfx95_dense_post_attention_norm(layer, x, ffn_coefficients)
mhc = moe_mhc_fusion(layer, residual, ffn_coefficients, forward_batch)
with use_mhc_post_fusion(mhc):
x = layer._run_moe_ffn_dp_sync(
x, forward_batch, input_ids=input_ids, input_ids_global=input_ids_global
)
x = layer._run_moe_ffn_dp_sync(
x,
forward_batch,
input_ids=input_ids,
input_ids_global=input_ids_global,
return_moe_output=mhc is not None,
)
ffn_pre, ffn_post, ffn_comb = ffn_coefficients.tensors()
if mhc is not None and mhc.output is not None:
if mhc is not None:
# The MoE TP path honors the deferred output under moe_mhc_fusion's gates;
# fail loud rather than fall back to an unfused reduce. The fused post
# carries the reduce, so take the merged rows without one.
assert isinstance(x, MoEOutput)
# Reduction already applied post. The next boundary consumes this
# materialized residual, including when it would normally defer post.
return mhc.output, ffn_pre, None
return apply_moe_mhc(x.get_merged(), mhc), ffn_pre, None
if defer_post:
return None, ffn_pre, (x, residual, ffn_post, ffn_comb)
return layer.hc_post(x, residual, ffn_post, ffn_comb), ffn_pre, None
@@ -167,20 +167,15 @@ def init_decoder_layer(layer, quant_config) -> None:
)
def hc_post(layer, x, residual, post, comb) -> Optional[torch.Tensor]:
def hc_post(cfg, x, residual, post, comb) -> Optional[torch.Tensor]:
"""V4.1's gfx950 hc_post at 768-4096 rows: the split-H kernel with 2048-wide blocks,
whose ordinary stores preserve locality for the following mHC reader; None elsewhere."""
if not (
_is_gfx95_supported
and layer.config.model_type == "deepseek_v41"
and cfg.pre_from_prev
and cfg.mult == 4
and 768 <= x.shape[0] <= 4096
and x.shape[1] == 5120
and residual.shape == (x.shape[0], 4, 5120)
and post.shape == (x.shape[0], 4)
and comb.shape == (x.shape[0], 4, 4)
and x.dtype == residual.dtype == torch.bfloat16
and post.dtype == comb.dtype == torch.float32
and all(t.is_contiguous() for t in (x, residual, post, comb))
):
return None
return mhc_post_split_h(x, residual, post, comb, block_size=2048)
@@ -216,14 +211,12 @@ def input_norm(
def forward_layer_fused_boundary(
model,
i: int,
state,
*,
positions: torch.Tensor,
hidden_states: Optional[torch.Tensor],
input_ids: torch.Tensor,
forward_batch,
input_ids_global: torch.Tensor,
prev_pre: Optional[torch.Tensor],
pending_post: Optional[Tuple[torch.Tensor, ...]],
capture_dspark: bool,
):
"""Layer i through the fused mHC boundary. Its FFN hc_post stays pending for the next
@@ -235,14 +228,23 @@ def forward_layer_fused_boundary(
and model.layers[nxt].engram is None
and not (capture_dspark and nxt in model.dspark_layers_to_capture)
)
return forward_hc_pre_from_prev_fused_boundary(
# The boundary speaks its own (hidden_states, prev_pre, pending_post) triple;
# the loop speaks HcState. This is the only place the two meet.
from sglang.srt.models.deepseek_v4_mhc import HcPending, HcState
pending = isinstance(state.streams, HcPending)
hidden_states, prev_pre, pending_post = forward_hc_pre_from_prev_fused_boundary(
model.layers[i],
positions=positions,
hidden_states=hidden_states,
hidden_states=None if pending else state.streams,
input_ids=input_ids,
forward_batch=forward_batch,
input_ids_global=input_ids_global,
prev_pre=prev_pre,
pending_post=pending_post,
prev_pre=state.pre,
pending_post=tuple(state.streams) if pending else None,
defer_post=defer_post,
)
return HcState(
hidden_states if pending_post is None else HcPending(*pending_post),
prev_pre,
)
+78 -147
View File
@@ -23,7 +23,17 @@ from __future__ import annotations
import logging
from contextlib import contextmanager, nullcontext
from functools import cached_property
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
from typing import (
Any,
Callable,
Dict,
Iterable,
List,
NamedTuple,
Optional,
Tuple,
Union,
)
import torch
import torch.nn.functional as F
@@ -96,7 +106,6 @@ from sglang.srt.layers.moe import (
get_moe_a2a_backend,
get_moe_runner_backend,
post_experts_all_reduce,
should_skip_post_experts_all_reduce,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
@@ -133,7 +142,6 @@ from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
Mxfp8RoutedInputPreQuant,
maybe_fuse_routed_scale_and_shared_add,
routed_hidden_size,
should_use_fuse_finalize_all_reduce,
)
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope_wrapper
@@ -582,9 +590,33 @@ class MoEGate(nn.Module):
return logits
# The dedicated 4 MiB push slot fits 384 rows of 5120 BF16 values.
# The dispatch gate also checks slot capacity and available counters.
_FUSED_FINALIZE_ALL_REDUCE_MAX_TOKENS = 384
class MoEOutput(NamedTuple):
"""A MoE block's pieces, before anything merges or reduces them. ``routed`` is
a deferred finalize handle or the raw routed rows."""
routed: Any
shared: Optional[torch.Tensor]
experts: nn.Module
routed_scaling_factor: float
shared_is_replicated: bool
def get_merged(self) -> torch.Tensor:
"""Merge the pieces, leaving the reduction to the caller."""
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
FlashInferTrtllmDeferredFinalizeOutput,
finalize_flashinfer_trtllm_deferred_output,
)
# NOTE: normal TP-sharded shared expert out
shared = None if self.shared_is_replicated else self.shared
if isinstance(self.routed, FlashInferTrtllmDeferredFinalizeOutput):
return finalize_flashinfer_trtllm_deferred_output(self.routed, shared)
return maybe_fuse_routed_scale_and_shared_add(
self.experts,
self.routed,
shared,
self.routed_scaling_factor,
)
class DeepseekV2MoE(nn.Module):
@@ -645,13 +677,6 @@ class DeepseekV2MoE(nn.Module):
self.routed_quant_stream = routed_quant_stream
self.is_nextn = is_nextn
self.is_deepseek_v4 = is_deepseek_v4
self._fuse_finalize_all_reduce = (
is_deepseek_v4
and getattr(config, "hc_pre_from_prev_sublayer", False)
and get_platform().is_blackwell
and self.tp_size == 4
)
n_hash_layers = getattr(config, "num_hash_layers", 0)
self.is_hash = layer_id < n_hash_layers and not (is_deepseek_v4 and is_nextn)
@@ -957,7 +982,11 @@ class DeepseekV2MoE(nn.Module):
input_ids: Optional[torch.Tensor] = None,
input_ids_global: Optional[torch.Tensor] = None,
skip_shared_experts: bool = False,
) -> torch.Tensor:
return_moe_output: bool = False,
) -> Union[torch.Tensor, MoEOutput]:
"""``return_moe_output`` asks for the pieces unmerged, as `MoEOutput`; only
the TP paths honor it. Distinct from `get_forward().defer_moe_finalize`,
the V2/V3 layer-boundary handoff."""
from sglang.srt.layers.moe.mega_moe import forward_mega_moe, should_use_mega_moe
if should_use_mega_moe(self, hidden_states):
@@ -998,6 +1027,7 @@ class DeepseekV2MoE(nn.Module):
input_ids_global=input_ids_global,
num_token_non_padded=num_token_non_padded,
use_vision_topk=use_vision_topk,
return_moe_output=return_moe_output,
)
else:
return self.forward_normal(
@@ -1008,6 +1038,7 @@ class DeepseekV2MoE(nn.Module):
skip_shared_experts=skip_shared_experts,
num_token_non_padded=num_token_non_padded,
use_vision_topk=use_vision_topk,
return_moe_output=return_moe_output,
)
else:
return self.forward_deepep(
@@ -1022,7 +1053,8 @@ class DeepseekV2MoE(nn.Module):
input_ids_global: Optional[torch.Tensor] = None,
num_token_non_padded: Optional[torch.Tensor] = None,
use_vision_topk: bool = False,
) -> torch.Tensor:
return_moe_output: bool = False,
) -> Union[torch.Tensor, MoEOutput]:
# Note(kpham-sgl): issue order satisfies 3 constraints:
# - no stream explosion: main (routed) issued before alt block -> capture reuses 1 alt stream;
# - PDL overlap: routed is the last main-stream kernel (fuses w/ residual add);
@@ -1103,22 +1135,7 @@ class DeepseekV2MoE(nn.Module):
)
ready = self.routed_quant_stream.record_event()
routed_pre_quant_input = Mxfp8RoutedInputPreQuant(x_q, x_sf, ready)
# The mHC post-split consumes the reduced row without an RMSNorm.
use_fused_finalize_all_reduce = (
self._fuse_finalize_all_reduce
# The kernel completes the tensor-parallel sum itself.
and self.reduce_results
and has_shared_output
and hidden_states.shape[-1] == 5120
and not self._shared_expert_tp1
and self.tp_size > 1
and hidden_states.shape[0] <= _FUSED_FINALIZE_ALL_REDUCE_MAX_TOKENS
and not should_skip_post_experts_all_reduce(is_tp_path=True)
and should_use_fuse_finalize_all_reduce(
self.experts, hidden_states.shape[0], hidden_states.shape[-1]
)
)
deferred_finalize = use_fused_finalize_all_reduce or (
deferred_finalize = (
has_shared_output
and not self._shared_expert_tp1
and topk_output.format == TopKOutputFormat.BYPASSED
@@ -1128,6 +1145,11 @@ class DeepseekV2MoE(nn.Module):
or hidden_states.shape[0] <= self._deferred_finalize_max_tokens
)
)
# NOTE: if `return_moe_output`, always try to defer finalize
deferred_finalize = deferred_finalize or (
return_moe_output
and isinstance(self.experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod)
)
if deferred_finalize:
final_hidden_states = self.experts.forward_deferred_finalize(
hidden_states, topk_output, pre_quant_input=routed_pre_quant_input
@@ -1174,91 +1196,24 @@ class DeepseekV2MoE(nn.Module):
reduce=sum_post_experts_output,
)
all_reduce_done = False
if return_moe_output:
return MoEOutput(
routed=final_hidden_states,
shared=shared_output,
experts=self.experts,
routed_scaling_factor=self.routed_scaling_factor,
shared_is_replicated=self._shared_expert_tp1,
)
if deferred_finalize:
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
finalize_flashinfer_trtllm_deferred_output,
)
deferred = final_hidden_states
if (
use_fused_finalize_all_reduce
and deferred.gemm2_out.shape[1] == hidden_states.shape[-1]
):
from sglang.kernels.ops.communication.all_reduce_fusion import (
moe_finalize_all_reduce,
)
from sglang.srt.layers.moe.mhc_post_fusion import (
current_mhc_post_fusion,
)
mhc = current_mhc_post_fusion()
if mhc is not None:
from sglang.kernels.ops.communication.all_reduce_mhc import (
moe_finalize_all_reduce_mhc,
)
# Join the coefficients before the fused epilogue reads them.
mhc.materialize_stats()
if mhc.stats_stream is not None:
current_stream.wait_stream(mhc.stats_stream)
args = (
deferred.gemm2_out,
deferred.expanded_idx_to_permuted_idx,
deferred.expert_weights,
deferred.top_k,
shared_output,
mhc.residual,
mhc.post,
mhc.comb,
)
if mhc.combine_only:
from sglang.kernels.ops.communication.all_reduce_mhc_combine import (
moe_finalize_all_reduce_mhc_combine,
)
final_hidden_states, mhc.output, mhc.combined = (
moe_finalize_all_reduce_mhc_combine(
*args, mhc.pre, world_size=self.tp_size
)
)
elif mhc.norm_weight is not None:
from sglang.kernels.ops.communication.all_reduce_mhc import (
moe_finalize_all_reduce_mhc_quant,
)
final_hidden_states, mhc.output, mhc.normalized, q, sf = (
moe_finalize_all_reduce_mhc_quant(
*args,
mhc.pre,
mhc.norm_weight,
mhc.norm_eps,
world_size=self.tp_size,
)
)
mhc.quantized = (q, sf)
else:
final_hidden_states, mhc.output = moe_finalize_all_reduce_mhc(
*args, world_size=self.tp_size
)
else:
final_hidden_states = moe_finalize_all_reduce(
deferred.gemm2_out,
deferred.expanded_idx_to_permuted_idx,
deferred.expert_weights,
deferred.top_k,
shared_output,
world_size=self.tp_size,
hidden_dim=hidden_states.shape[-1],
# Routing metadata must be ready before it is consumed.
prefetch_metadata=False,
)
all_reduce_done = True
else:
final_hidden_states = finalize_flashinfer_trtllm_deferred_output(
deferred,
shared_output,
)
final_hidden_states = finalize_flashinfer_trtllm_deferred_output(
final_hidden_states,
shared_output,
)
else:
final_hidden_states = maybe_fuse_routed_scale_and_shared_add(
self.experts,
@@ -1266,24 +1221,7 @@ class DeepseekV2MoE(nn.Module):
None if self._shared_expert_tp1 else shared_output,
self.routed_scaling_factor,
)
if not all_reduce_done and self.reduce_results:
if (
self.is_deepseek_v4
and self.tp_size > 1
and not should_skip_post_experts_all_reduce(is_tp_path=True)
):
from sglang.srt.layers.moe.mhc_post_fusion import (
current_mhc_post_fusion,
)
mhc = current_mhc_post_fusion()
if _is_hip and _hip_moe.fused_all_reduce_mhc(
self, mhc, final_hidden_states
):
return final_hidden_states
if mhc is not None:
mhc.start_stats_before_all_reduce()
if self.reduce_results:
final_hidden_states = post_experts_all_reduce(final_hidden_states)
# TP1 shared experts are replicated: add them after this block's
# all-reduce, or on TP rank 0 only when the stage boundary sums.
@@ -1300,7 +1238,8 @@ class DeepseekV2MoE(nn.Module):
skip_shared_experts: bool = False,
num_token_non_padded: Optional[torch.Tensor] = None,
use_vision_topk: bool = False,
) -> torch.Tensor:
return_moe_output: bool = False,
) -> Union[torch.Tensor, MoEOutput]:
if hasattr(self, "shared_experts") and use_intel_amx_backend(
self.shared_experts.gate_up_proj
):
@@ -1435,30 +1374,22 @@ class DeepseekV2MoE(nn.Module):
pre_quant_input=pre_quant_input,
)
if return_moe_output:
return MoEOutput(
routed=final_hidden_states,
shared=shared_output,
experts=self.experts,
routed_scaling_factor=self.routed_scaling_factor,
shared_is_replicated=self._shared_expert_tp1,
)
final_hidden_states = maybe_fuse_routed_scale_and_shared_add(
self.experts,
final_hidden_states,
None if self._shared_expert_tp1 else shared_output,
self.routed_scaling_factor,
)
if self.reduce_results:
if (
self.is_deepseek_v4
and self.tp_size > 1
and not should_skip_post_experts_all_reduce(is_tp_path=True)
):
from sglang.srt.layers.moe.mhc_post_fusion import (
current_mhc_post_fusion,
)
mhc = current_mhc_post_fusion()
if _is_hip and _hip_moe.fused_all_reduce_mhc(
self, mhc, final_hidden_states
):
return final_hidden_states
if mhc is not None:
mhc.start_stats_before_all_reduce()
final_hidden_states = post_experts_all_reduce(final_hidden_states)
# TP1 shared experts are replicated: add them after this block's
# all-reduce, or on TP rank 0 only when the stage boundary sums.
File diff suppressed because it is too large Load Diff
+16 -19
View File
@@ -35,6 +35,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models import deepseek_v4_mhc as mhc
from sglang.srt.models.dbrx import ReplicatedLinear
from sglang.srt.models.deepseek_v4 import (
DEEPSEEK_V4_STACKED_PARAMS_MAPPING,
@@ -745,35 +746,31 @@ class DSparkV4Stage(DeepseekV4DecoderLayer):
forward_batch: ForwardBatch,
prev_pre: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
stats_stream = self._get_hc_stats_stream(hidden_states, forward_batch)
# The draft stages run the plain four steps: no fused post, no seams, so
# they need nothing from `bind_next` and never hand anything forward.
assert self.attn_hc is not None and self.ffn_hc is not None
stats_stream = None
if mhc.use_stats_stream(self.hc_cfg, forward_batch, hidden_states):
stats_stream = self.hc_stats_stream
residual = hidden_states
x = self._hc_combine(
hidden_states, prev_pre, self.input_layernorm, stats_stream
)
mhc.fork_stats_stream(stats_stream)
x = mhc.combine(self.attn_hc, mhc.HcState(hidden_states, prev_pre))
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
x = self.self_attn(positions, x, forward_batch)
attn_pre, attn_post, attn_comb = self._hc_mix_stats(
hidden_states,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
stats_stream,
attn_pre, attn_post, attn_comb = mhc.mix_stats(
self.attn_hc, hidden_states, stats_stream
)
if stats_stream is not None:
torch.cuda.current_stream().wait_stream(stats_stream)
hidden_states = self.hc_post(x, residual, attn_post, attn_comb)
residual = hidden_states
x = self._hc_combine(
hidden_states, attn_pre, self.post_attention_layernorm, stats_stream
)
mhc.fork_stats_stream(stats_stream)
x = mhc.combine(self.ffn_hc, mhc.HcState(hidden_states, attn_pre))
x = self._run_ffn(x, forward_batch)
ffn_pre, ffn_post, ffn_comb = self._hc_mix_stats(
hidden_states,
self.hc_ffn_fn,
self.hc_ffn_scale,
self.hc_ffn_base,
stats_stream,
ffn_pre, ffn_post, ffn_comb = mhc.mix_stats(
self.ffn_hc, hidden_states, stats_stream
)
if stats_stream is not None:
torch.cuda.current_stream().wait_stream(stats_stream)
+715
View File
@@ -0,0 +1,715 @@
"""DeepSeek-V4.1 manifold-constrained hyper-connections (mHC).
The residual is ``R [T, hc_mult, hidden]`` -- hc_mult parallel streams per token --
and each sublayer is wrapped in a hyper-connection (DeepSeek's ``attn_hc`` and
``ffn_hc``) carrying a ``(pre, post, comb)`` triplet. Per hyper-connection:
1. mix stats predict ``(pre, post, comb)`` from ``R``
2. combine collapse the streams with the PREVIOUS triplet's ``pre``, then norm
3. sublayer attention or MoE, including its tensor-parallel reduction
4. post ``R'[i] = post[i] * y + sum_j comb[j][i] * R[j]``
Step 1's ``pre`` is consumed by step 2 of the *next* hyper-connection -- the lag
that lets step 1 run beside steps 2-3 and makes V4's fused post+pre inexpressible.
``attn_hc -> ffn_hc`` stays inside a layer; ``ffn_hc -> attn_hc`` crosses layers,
where an Engram or a row selection can replace ``R`` (`with_residual`/`take_rows`).
The layer owns parameters, parallelism and token-range policy; this module owns
the kernel dispatch.
TODO: move dsv4 MHC into the same abstraction
"""
from __future__ import annotations
from typing import Any, Callable, NamedTuple, Optional, Tuple, TypeAlias, Union
import msgspec
import torch
import torch.nn.functional as F
from sglang.kernels.ops.layernorm.mhc_post_split_h import mhc_post_split_h
from sglang.srt.environ import envs
from sglang.srt.hardware_backend.npu.utils import is_npu_arch35
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.quantization.mxfp8_input import Mxfp8SwizzledInput
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.runner import compile_in_capture_mode
from sglang.srt.models.deepseek_v2 import MoEOutput, _is_hip, _is_npu, _is_xpu
from sglang.srt.runtime_context import get_parallel, get_platform
from sglang.srt.utils import is_gfx95_supported
_is_gfx95_supported = is_gfx95_supported()
# ---------------------------------------------------------------------------
# What a layer hands this module
# ---------------------------------------------------------------------------
HcTriplet = Tuple[torch.Tensor, torch.Tensor, torch.Tensor]
class HcConfig(NamedTuple):
"""Everything about a layer's mHC that is fixed for the model's lifetime."""
mult: int
sinkhorn_iters: int
eps: float
rms_eps: float
hidden: int
pre_from_prev: bool
cp_prefill: bool
class HcSubLayer(NamedTuple):
"""One hyper-connection's fixed material. Holds the layer's Parameters by
reference; a plain tuple registers nothing, so checkpoint keys stay put."""
cfg: HcConfig
fn: torch.Tensor
scale: torch.Tensor
base: torch.Tensor
norm: RMSNorm
tf32_parts: Optional[Any] = None
bf16_parts: Optional[Any] = None
# ---------------------------------------------------------------------------
# How far the previous hyper-connection got toward the next sublayer's input
# ---------------------------------------------------------------------------
class HcNormed(NamedTuple):
"""Summed and normed; the sublayer can take these rows as they are."""
rows: torch.Tensor
class HcQuantized(NamedTuple):
"""Normed, and already swizzled for the sublayer's first projection."""
rows: torch.Tensor
swizzled: Mxfp8SwizzledInput
HcPreOutput: TypeAlias = Union[HcNormed, HcQuantized]
"""The next sublayer's input, built by the previous post; a receiver skips step 2
(and ``pre``) entirely. Always normed -- combine and norm travel together."""
class AttnOutput(NamedTuple):
"""Attention's wo_b rows with the TP reduction left to the fused post."""
partial: torch.Tensor
# ---------------------------------------------------------------------------
# The real parameters: what crosses from one hyper-connection to the next
# ---------------------------------------------------------------------------
class HcPending(NamedTuple):
"""A post not run yet; the residual it would produce is implied. The backward
counterpart of `HcPreOutput`: the HIP boundary kernel takes both and runs the
previous post, this triplet and this collapse in one launch."""
rows: torch.Tensor
residual: torch.Tensor
post: torch.Tensor
comb: torch.Tensor
class HcState(msgspec.Struct):
"""The mHC state of the stream between two hyper-connections.
``streams`` is the residual ``R``, or -- out of the HIP fused boundary -- the
post that would rebuild it, still outstanding. Mutable for one reason:
`release` must empty a consumed state in every frame that still binds it.
"""
streams: Union[torch.Tensor, HcPending, None]
pre: Optional[torch.Tensor] = None
input: Optional[HcPreOutput] = None
def release(self) -> None:
"""Empty the state at its last reader, the combine; the residual the post
still needs lives on through the caller's alias. Stale holders -- above
all the model loop's binding across the layer call -- then stop pinning
the dead residual through the next sublayer."""
self.streams = None
self.pre = None
self.input = None
@property
def residual(self) -> torch.Tensor:
"""The materialized streams; run an outstanding post first (`materialized`)."""
assert isinstance(self.streams, torch.Tensor)
return self.streams
def materialized(self, cfg: HcConfig) -> HcState:
"""Run the outstanding post, if any; residual readers call this first."""
if not isinstance(self.streams, HcPending):
return self
p = self.streams
return HcState(post(cfg, p.rows, p.residual, p.post, p.comb), self.pre)
def with_residual(self, residual: torch.Tensor) -> HcState:
"""The residual was rewritten: drop ``input``; ``pre`` weights streams, not
rows, so it survives."""
return HcState(residual, self.pre)
def take_rows(self, rows: Callable[[torch.Tensor], torch.Tensor]) -> HcState:
"""Keep only the rows ``rows`` selects -- the late-layer tail's narrowing."""
pre = None if self.pre is None else rows(self.pre)
assert self.streams is not None
if isinstance(self.streams, HcPending):
return HcState(HcPending(*(rows(t) for t in self.streams)), pre)
else:
return HcState(rows(self.streams), pre)
# ---------------------------------------------------------------------------
# The plan: how a hyper-connection intends to run its post
# ---------------------------------------------------------------------------
class HcNextBoundary(NamedTuple):
"""What the next hyper-connection can take off this one's post; resolved after
weights load. None where the seam is closed (last layer, Engram)."""
norm: RMSNorm
accepts_mxfp8: bool
norm_fusable: bool
def is_deferred_finalize(routed) -> bool:
"""Whether a `MoEOutput` really carries an unfinalized handle for the post."""
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
FlashInferTrtllmDeferredFinalizeOutput,
)
return isinstance(routed, FlashInferTrtllmDeferredFinalizeOutput)
def make_boundary(norm: RMSNorm, *, accepts_mxfp8: bool) -> HcNextBoundary:
"""Can this hyper-connection's norm fold into the previous post, and does its
sublayer take a pre-quantized input."""
return HcNextBoundary(
norm=norm,
accepts_mxfp8=accepts_mxfp8,
norm_fusable=(
not norm.cast_x_before_out_mul
and norm.variance_size_override is None
and norm.weight.dtype == torch.bfloat16
# TODO: support other norm shape
and norm.weight.shape == (5120,)
and norm.weight.is_contiguous()
),
)
# ---------------------------------------------------------------------------
# Step 1 -- the mix GEMM and the triplet it splits into
# ---------------------------------------------------------------------------
def use_stats_stream(cfg: HcConfig, forward_batch: ForwardBatch, x: torch.Tensor):
if not cfg.pre_from_prev:
return False
decode_like = forward_batch.forward_mode.is_decode() or (
forward_batch.forward_mode.is_target_verify() and x.shape[0] > 0
)
return decode_like and (
not get_platform().is_sm90
or x.shape[0] == 1
or (forward_batch.forward_mode.is_decode() and 1 < x.shape[0] <= 64)
)
def mix_stats(
hc: HcSubLayer, x: torch.Tensor, stats_stream: Optional[torch.cuda.Stream] = None
) -> HcTriplet:
"""Predict the triplet on the caller's stream or the stats stream."""
if stats_stream is None:
return _mix_stats_impl(hc, x)
main_stream = torch.cuda.current_stream()
x.record_stream(stats_stream)
with torch.cuda.stream(stats_stream):
coefficients = _mix_stats_impl(hc, x)
for coefficient in coefficients:
coefficient.record_stream(main_stream)
return coefficients
def _mix_stats_impl(hc: HcSubLayer, x: torch.Tensor) -> HcTriplet:
from sglang.kernels.ops.layernorm.mhc import (
hc_mix_stats,
hc_mix_stats_sinkhorn,
hc_split_sinkhorn,
)
cfg = hc.cfg
x_flat = x.flatten(1)
from sglang.srt.batch_invariant_ops import (
is_batch_invariant_mode_enabled,
)
parts = bf16_parts = None
hopper_medium = get_platform().is_sm90 and 32 <= x_flat.shape[0] < 4096
if (
x.is_cuda
and (x_flat.shape[0] >= 128 or hopper_medium)
and x_flat.is_contiguous()
and (get_platform().is_sm100 or get_platform().is_sm90)
and envs.SGLANG_OPT_DEEPGEMM_HC_PRENORM.get()
and not is_batch_invariant_mode_enabled()
):
parts, bf16_parts = hc.tf32_parts, hc.bf16_parts
use_bf16_projection = bf16_parts is not None and (
hopper_medium or 4096 <= x_flat.shape[0] <= 65536
)
hopper_fused_stats = get_platform().is_sm90 and (
x.shape[0] == 1 or (bf16_parts is not None and 32 <= x.shape[0] <= 65536)
)
# gfx950 uses the fused Triton port at every row count (MI350X).
use_fused_stats = _is_gfx95_supported or (
torch.version.cuda is not None
and (get_platform().is_blackwell or hopper_fused_stats)
)
if x.is_cuda and use_fused_stats and x.dtype == torch.bfloat16:
# The default split-K/Sinkhorn fusion preserves batch invariance;
# compensated projections above are disabled in batch-invariant mode.
if use_bf16_projection:
from sglang.kernels.ops.layernorm.mhc import (
hc_mix_stats_sinkhorn_bf16x3,
)
pre, post, comb = hc_mix_stats_sinkhorn_bf16x3(
x_flat,
bf16_parts,
hc.scale,
hc.base,
cfg.sinkhorn_iters,
cfg.rms_eps,
cfg.eps,
)
elif parts is not None:
from sglang.kernels.ops.layernorm.mhc import (
hc_mix_stats_sinkhorn_deepgemm,
)
pre, post, comb = hc_mix_stats_sinkhorn_deepgemm(
x_flat,
parts,
hc.scale,
hc.base,
cfg.sinkhorn_iters,
cfg.rms_eps,
cfg.eps,
)
else:
pre, post, comb = hc_mix_stats_sinkhorn(
x_flat,
hc.fn,
hc.scale,
hc.base,
cfg.mult,
cfg.sinkhorn_iters,
cfg.rms_eps,
cfg.eps,
)
return pre, post, comb
if x.is_cuda and torch.version.cuda is not None:
# cuBLAS/torch reductions can change order with num_tokens; this kernel
# keeps the mixing and RMS reductions batch-invariant.
mixes = hc_mix_stats(x_flat, hc.fn, cfg.rms_eps).unsqueeze(1)
else:
x_flat = x_flat.float()
rsqrt = torch.rsqrt(x_flat.square().mean(-1, keepdim=True) + cfg.rms_eps)
mixes = (F.linear(x_flat, hc.fn) * rsqrt).unsqueeze(1)
if _is_xpu:
from sglang.srt.models.deepseek_v4 import get_mhc_ops
split_sinkhorn = get_mhc_ops().hc_split_sinkhorn
else:
split_sinkhorn = hc_split_sinkhorn
pre, post, comb = split_sinkhorn(
mixes,
hc.scale,
hc.base,
cfg.mult,
cfg.sinkhorn_iters,
cfg.eps,
)
return pre.squeeze(1), post.squeeze(1), comb.squeeze(1)
# ---------------------------------------------------------------------------
# Step 2 -- collapse the streams and normalize
# ---------------------------------------------------------------------------
def fork_stats_stream(stats_stream: Optional[torch.cuda.Stream]) -> None:
"""Let the side stream pick up from here; the layers call it right before
each combine (fork position measured indistinguishable across batch sizes)."""
if stats_stream is not None:
stats_stream.wait_stream(torch.cuda.current_stream())
def combine(
hc: HcSubLayer,
state: HcState,
quantized: Optional[list] = None,
) -> torch.Tensor:
"""The sublayer's input: ``norm(sum_k pre[k] * R[k])``, short-circuited by
whatever the previous post left in ``state.input``."""
shortcut = state.input
if isinstance(shortcut, HcQuantized):
assert quantized is not None
quantized.append(shortcut.swizzled)
return shortcut.rows
if isinstance(shortcut, HcNormed):
return shortcut.rows
return _combine(hc, state, quantized)
def _combine(
hc: HcSubLayer,
state: HcState,
quantized: Optional[list],
) -> torch.Tensor:
"""The combine computed from the residual's rows."""
from sglang.kernels.ops.layernorm.mhc import hc_combine
cfg, norm = hc.cfg, hc.norm
x, apply_pre = state.residual, state.pre
quantize = quantized is not None
x_flat = x.flatten(1)
if apply_pre is None:
return norm(x[:, 0, :])
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
hopper_fused = (
get_platform().is_sm90
and not quantize
and (0 < x.shape[0] <= 96 or 4096 <= x.shape[0] <= 65536)
and not is_batch_invariant_mode_enabled()
)
if (
x.is_cuda
and (get_platform().is_blackwell or hopper_fused)
and x.dtype == torch.bfloat16
and apply_pre.stride(1) == 1
and cfg.mult == 4
and cfg.hidden == 5120
and norm.weight.dtype == torch.bfloat16
and not norm.cast_x_before_out_mul
and norm.variance_size_override is None
):
# One fused form for every row count. Each row's reduction order is
# fixed regardless of the grid split, so this holds under
# batch-invariant mode too.
if quantize and x.shape[0] <= 128:
from sglang.kernels.ops.layernorm.hc_combine_norm import (
hc_combine_norm_mxfp8,
)
y, y_q, y_sf = hc_combine_norm_mxfp8(
x_flat, apply_pre, norm.weight, norm.variance_epsilon
)
quantized.append(Mxfp8SwizzledInput(y_q, y_sf))
return y
from sglang.kernels.ops.layernorm.hc_combine_norm import hc_combine_norm
return hc_combine_norm(x_flat, apply_pre, norm.weight, norm.variance_epsilon)
return norm(hc_combine(x_flat, apply_pre, cfg.mult, x.dtype))
# ---------------------------------------------------------------------------
# Step 4 -- rebuild the streams, optionally collapsing the next input too
# ---------------------------------------------------------------------------
def post(
cfg: HcConfig,
x: torch.Tensor,
residual: torch.Tensor,
post_mix: torch.Tensor,
comb: torch.Tensor,
) -> torch.Tensor:
"""``R'[i] = post[i] * x + sum_j comb[j][i] * R[j]``, by whichever kernel fits."""
if x.shape[0] == 0:
return torch.empty((0, cfg.mult, x.shape[-1]), dtype=x.dtype, device=x.device)
if _is_npu:
if not is_npu_arch35():
return torch.ops.custom.npu_hc_post(x, residual, post_mix, comb)
# The A5 build of npu_hc_post is batched -- it requires a leading batch axis
# on every operand.
return torch.ops.custom.npu_hc_post(
x.unsqueeze(0),
residual.unsqueeze(0),
post_mix.unsqueeze(0),
comb.unsqueeze(0),
).squeeze(0)
if _is_xpu:
from sglang.srt.models.deepseek_v4 import get_mhc_ops
return get_mhc_ops().mhc_post(x, residual, post_mix, comb)
if (
get_platform().is_blackwell
and cfg.pre_from_prev
and cfg.mult == 4
and cfg.hidden == 5120
# This 384 is the kernel's own tile limit, unrelated to the collective's
# push slots; it covers every narrow batch, not one particular regime.
and x.shape[0] <= 384
):
return mhc_post_split_h(x, residual, post_mix, comb)
if _is_hip:
from sglang.srt.models.deepseek_common.amd import deepseek_v4_hip as _hip
y = _hip.hc_post(cfg, x, residual, post_mix, comb)
if y is not None:
return y
if envs.SGLANG_OPT_USE_FLASHINFER_MHC.get():
from flashinfer.mhc import mhc_post
return mhc_post(x, residual, post_mix, comb)
if envs.SGLANG_OPT_USE_TILELANG_MHC_POST.get():
if (
cfg.pre_from_prev
and get_platform().is_sm90
and cfg.mult == 4
and cfg.hidden == 5120
and 1 <= x.shape[0] <= 64
):
return mhc_post_split_h(x, residual, post_mix, comb)
from sglang.kernels.ops.layernorm.mhc import mhc_post
return mhc_post(x, residual, post_mix, comb)
if _is_hip:
from aiter.ops.mhc import mhc_post
result = torch.empty_like(residual)
mhc_post(result, x, residual, post_mix, comb)
return result
assert residual.shape == (x.shape[0], cfg.mult, x.shape[-1])
assert post_mix.shape == (x.shape[0], cfg.mult)
assert comb.shape == (x.shape[0], cfg.mult, cfg.mult)
@compile_in_capture_mode
def post_torch_impl(x, residual, post_mix, comb):
return (
post_mix.unsqueeze(-1) * x.unsqueeze(1)
+ (comb.unsqueeze(-1) * residual.unsqueeze(2)).sum(dim=1)
).type_as(x)
return post_torch_impl(x, residual, post_mix, comb)
# ---------------------------------------------------------------------------
# Planning the post
# ---------------------------------------------------------------------------
def can_fuse_post(cfg: HcConfig) -> bool:
"""Whether a fused post is expressible at all for this model, device and
parallel layout.
The capability half of the decision; which token ranges should take it, and what
batch-invariant mode should do about it, are the caller's policy.
"""
return (
get_platform().is_blackwell
and cfg.pre_from_prev
and cfg.mult == 4
and cfg.hidden == 5120
and get_parallel().attn_dp_size == 1
and not cfg.cp_prefill
)
# ---------------------------------------------------------------------------
# Running the post
# ---------------------------------------------------------------------------
def _compute_triplet(
hc: HcSubLayer,
residual: torch.Tensor,
stats_stream: Optional[torch.cuda.Stream],
) -> HcTriplet:
"""Issue the triplet (on the side stream, beside the sublayer) and join before
the post reads it."""
coefficients = mix_stats(hc, residual, stats_stream)
if stats_stream is not None:
torch.cuda.current_stream().wait_stream(stats_stream)
return coefficients
def _post_fusion(
hc: HcSubLayer,
y: torch.Tensor,
residual: torch.Tensor,
coefficients: HcTriplet,
next: Optional[HcNextBoundary],
) -> HcState:
"""Step 4 without a collective: the wide-tile kernel folds the next combine +
norm in where it serves this seam, otherwise the pure post runs and the next
combine computes itself."""
cfg = hc.cfg
pre, post_mix, comb = coefficients
if (
next is not None
and next.norm_fusable
and cfg.pre_from_prev
and get_platform().is_blackwell
and 4096 <= y.shape[0] <= 65536
and cfg.hidden == 5120
and cfg.mult == 4
and get_parallel().attn_dp_size == 1
and not cfg.cp_prefill
):
from sglang.kernels.ops.layernorm.mhc_post_combine_norm_prefill import (
mhc_post_combine_norm_prefill,
)
updated, normalized = mhc_post_combine_norm_prefill(
y,
residual,
post_mix,
comb,
pre,
next.norm.weight,
next.norm.variance_epsilon,
)
return HcState(updated, pre, HcNormed(normalized))
return HcState(post(cfg, y, residual, post_mix, comb), pre)
def run_attn_post(
hc: HcSubLayer,
out: Union[torch.Tensor, AttnOutput],
residual: torch.Tensor,
*,
stats_stream: Optional[torch.cuda.Stream],
next: Optional[HcNextBoundary],
world_size: int,
) -> HcState:
"""The attention post. An `AttnOutput` rides the collective kernel (which also
folds ``next``'s norm); the attention may decline the handover even when asked,
so the type is the ground truth."""
if isinstance(out, AttnOutput):
from sglang.kernels.ops.communication.all_reduce_mhc import (
all_reduce_mhc_post_combine_norm,
)
assert next is not None
pre, post_mix, comb = _compute_triplet(hc, residual, stats_stream)
_, updated, normalized = all_reduce_mhc_post_combine_norm(
out.partial,
residual,
post_mix,
comb,
pre,
next.norm.weight,
next.norm.variance_epsilon,
world_size=world_size,
)
return HcState(updated, pre, HcNormed(normalized))
coefficients = _compute_triplet(hc, residual, stats_stream)
return _post_fusion(hc, out, residual, coefficients, next)
def run_moe_post(
hc: HcSubLayer,
out: Union[torch.Tensor, MoEOutput],
residual: torch.Tensor,
*,
stats_stream: Optional[torch.cuda.Stream],
next: Optional[HcNextBoundary],
world_size: int,
) -> HcState:
"""The MoE post. A deferred finalize the push plane can carry rides the
collective kernel (finalize + shared add + all-reduce, quantizing ``next``'s
input when a boundary is given); anything else is finalized here and takes
the plain post."""
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
can_fuse_all_reduce,
)
if (
isinstance(out, MoEOutput)
and is_deferred_finalize(out.routed)
# expert_weights rows are the true token count; gemm2_out is the expanded
# [T x top_k (padded)] view and overstates the plane load by ~6x.
and can_fuse_all_reduce(out.routed.expert_weights.shape[0], hc.cfg.hidden)
):
pre, post_mix, comb = _compute_triplet(hc, residual, stats_stream)
args = (
out.routed.gemm2_out,
out.routed.expanded_idx_to_permuted_idx,
out.routed.expert_weights,
out.routed.top_k,
out.shared,
residual,
post_mix,
comb,
)
if (
next is not None
and next.accepts_mxfp8
and next.norm_fusable
# The quant epilogue supports at most 128 batch size
and out.routed.expert_weights.shape[0] <= 128
):
from sglang.kernels.ops.communication.all_reduce_mhc import (
moe_finalize_all_reduce_mhc_post_combine_norm_quant,
)
_, updated, normalized, q, sf = (
moe_finalize_all_reduce_mhc_post_combine_norm_quant(
*args,
pre,
next.norm.weight,
next.norm.variance_epsilon,
world_size=world_size,
)
)
return HcState(
updated, pre, HcQuantized(normalized, Mxfp8SwizzledInput(q, sf))
)
from sglang.kernels.ops.communication.all_reduce_mhc import (
moe_finalize_all_reduce_mhc_post,
)
_, updated = moe_finalize_all_reduce_mhc_post(*args, world_size=world_size)
return HcState(updated, pre)
if isinstance(out, MoEOutput):
from sglang.srt.layers.moe import post_experts_all_reduce
from sglang.srt.layers.moe.utils import should_add_replicated_moe_output
# The post boundary owns the sum; a replicated shared expert is added
# after the reduction so it is counted only once across TP ranks.
pieces = out
out = post_experts_all_reduce(pieces.get_merged())
if (
pieces.shared is not None
and pieces.shared_is_replicated
and should_add_replicated_moe_output()
):
out += pieces.shared
coefficients = _compute_triplet(hc, residual, stats_stream)
return _post_fusion(hc, out, residual, coefficients, next)