From 730f1f3e5be9c781f523f856fc39c18c2ad266d6 Mon Sep 17 00:00:00 2001 From: DarkSharpness <76582120+DarkSharpness@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:01:42 +0800 Subject: [PATCH] [DeepSeek-V4.1] Unify mHC into one state machine and drop medium-batch fusion variants (#42245) Co-authored-by: Claude Fable 5 Co-authored-by: BBuf <1182563586@qq.com> --- .../csrc/distributed/all_reduce_fusion.cuh | 42 +- .../ops/communication/all_reduce_mhc.py | 50 +- .../communication/all_reduce_mhc_combine.py | 121 -- .../kernels/ops/layernorm/hc_combine_norm.py | 9 +- .../kernels/ops/layernorm/mhc_post_combine.py | 85 -- .../sglang/srt/layers/moe/mhc_post_fusion.py | 57 - .../moe/moe_runner/flashinfer_trtllm.py | 2 +- .../mxfp4_flashinfer_trtllm_moe.py | 101 +- .../amd/deepseek_v2_hip_moe.py | 30 - .../amd/deepseek_v4_fused_mhc.py | 88 +- .../deepseek_common/amd/deepseek_v4_hip.py | 32 +- python/sglang/srt/models/deepseek_v2.py | 225 ++-- python/sglang/srt/models/deepseek_v4.py | 1092 ++++------------- .../sglang/srt/models/deepseek_v4_dspark.py | 35 +- python/sglang/srt/models/deepseek_v4_mhc.py | 715 +++++++++++ .../ops/layernorm/test_mhc_aiter_hip.py | 5 +- .../kernels/ops/layernorm/test_mhc_kernels.py | 128 +- .../ops/layernorm/test_mxfp8_epilogue.py | 4 +- .../unit/models/test_deepseek_v4_amd_tp4.py | 41 +- 19 files changed, 1359 insertions(+), 1503 deletions(-) delete mode 100644 python/sglang/kernels/ops/communication/all_reduce_mhc_combine.py delete mode 100644 python/sglang/kernels/ops/layernorm/mhc_post_combine.py delete mode 100644 python/sglang/srt/layers/moe/mhc_post_fusion.py create mode 100644 python/sglang/srt/models/deepseek_v4_mhc.py diff --git a/python/sglang/kernels/jit/csrc/distributed/all_reduce_fusion.cuh b/python/sglang/kernels/jit/csrc/distributed/all_reduce_fusion.cuh index 175d2beb8e..123dd2b126 100644 --- a/python/sglang/kernels/jit/csrc/distributed/all_reduce_fusion.cuh +++ b/python/sglang/kernels/jit/csrc/distributed/all_reduce_fusion.cuh @@ -510,7 +510,7 @@ struct MoeFinalizeAllReduceKernel { } /// Finalize + all-reduce + HC=4 post; original reduced output is retained. - static void run_mhc( + static void run_mhc_post( CommunicatorRef ref, TensorView out, TensorView gemm2_out, @@ -538,41 +538,7 @@ struct MoeFinalizeAllReduceKernel { comb); } - // Keep the original RMSNorm as a separate kernel, while reusing the - // BF16-rounded post values for the next sublayer's pre-combine. - static void run_mhc_combine( - CommunicatorRef ref, - TensorView out, - TensorView gemm2_out, - TensorView permuted_idx, - TensorView expert_weights, - std::optional shared_output, - TensorView mhc_out, - TensorView residual, - TensorView post, - TensorView comb, - TensorView pre, - TensorView combined) { - static_assert(kMhc && kCollapse && !kQuant); - run_impl( - ref, - out, - gemm2_out, - permuted_idx, - expert_weights, - shared_output, - std::nullopt, - 0.0, - false, - mhc_out, - residual, - post, - comb, - pre, - combined); - } - - static void run_mhc_norm( + static void run_mhc_post_combine_norm( CommunicatorRef ref, TensorView out, TensorView gemm2_out, @@ -606,7 +572,7 @@ struct MoeFinalizeAllReduceKernel { normalized); } - static void run_mhc_quant( + static void run_mhc_post_combine_norm_quant( CommunicatorRef ref, TensorView out, TensorView gemm2_out, @@ -707,7 +673,7 @@ struct MoeFinalizeAllReduceKernel { } const auto num_tokens = static_cast(T.unwrap()); if constexpr (kQuant) { - CHECK_HOST(num_tokens <= 8); + CHECK_HOST(num_tokens <= 128) << "the quant epilogue writes one 128-row scale tile"; CHECK_HOST(norm_weight.has_value()); TensorMatcher({T, kHiddenDim}).with_dtype().with_device(device).verify(quantized.value()); TensorMatcher({(kHiddenDim / 32) * 128}) diff --git a/python/sglang/kernels/ops/communication/all_reduce_mhc.py b/python/sglang/kernels/ops/communication/all_reduce_mhc.py index ad32c99a41..2c341ee62e 100644 --- a/python/sglang/kernels/ops/communication/all_reduce_mhc.py +++ b/python/sglang/kernels/ops/communication/all_reduce_mhc.py @@ -34,18 +34,21 @@ def _jit_mhc_module(world_size, top_k, cluster_size, weight_dtype): True, ) return load_jit( - "moe_finalize_all_reduce_mhc", + "moe_finalize_all_reduce_mhc_post", *args, cuda_files=["distributed/all_reduce_fusion.cuh"], cuda_wrappers=[ - ("run", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc"), - ("run_norm", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_norm"), + ("run", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_post"), + ( + "run_norm", + f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_post_combine_norm", + ), ], ) @register_custom_op(mutates_args=["out", "mhc_out"]) -def _moe_finalize_all_reduce_mhc_op( +def _moe_finalize_all_reduce_mhc_post_op( world_size: int, top_k: int, cluster_size: int, @@ -75,7 +78,7 @@ def _moe_finalize_all_reduce_mhc_op( ) -def moe_finalize_all_reduce_mhc( +def moe_finalize_all_reduce_mhc_post( gemm2: torch.Tensor, idx: torch.Tensor, weights: torch.Tensor, @@ -99,7 +102,7 @@ def moe_finalize_all_reduce_mhc( ) mhc_out = torch.empty_like(residual) if weights.shape[0]: - _moe_finalize_all_reduce_mhc_op( + _moe_finalize_all_reduce_mhc_post_op( world_size, top_k, cluster_size or default_cluster_size(_MHC_HIDDEN_DIM), @@ -117,7 +120,7 @@ def moe_finalize_all_reduce_mhc( @register_custom_op(mutates_args=["out", "mhc_out", "normalized"]) -def _moe_finalize_all_reduce_mhc_norm_op( +def _moe_finalize_all_reduce_mhc_post_combine_norm_op( world_size: int, top_k: int, cluster_size: int, @@ -155,7 +158,7 @@ def _moe_finalize_all_reduce_mhc_norm_op( ) -def moe_finalize_all_reduce_mhc_norm( +def moe_finalize_all_reduce_mhc_post_combine_norm( gemm2: torch.Tensor, idx: torch.Tensor, weights: torch.Tensor, @@ -171,7 +174,7 @@ def moe_finalize_all_reduce_mhc_norm( world_size: int, cluster_size: Optional[int] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """:func:`moe_finalize_all_reduce_mhc` plus the HC=4 pre-collapse + """:func:`moe_finalize_all_reduce_mhc_post` plus the HC=4 pre-collapse (``pre`` ``[T, 4]`` fp32) and RMSNorm of the collapsed row. Returns ``(reduced, mhc_out, normalized [T, hidden])``. """ @@ -181,7 +184,7 @@ def moe_finalize_all_reduce_mhc_norm( mhc_out = torch.empty_like(residual) normalized = torch.empty_like(out) if weights.shape[0]: - _moe_finalize_all_reduce_mhc_norm_op( + _moe_finalize_all_reduce_mhc_post_combine_norm_op( world_size, top_k, cluster_size or default_cluster_size(_MHC_HIDDEN_DIM), @@ -210,7 +213,7 @@ def _identity_routing(rows, device): ) -def all_reduce_mhc_norm( +def all_reduce_mhc_post_combine_norm( x: torch.Tensor, residual: torch.Tensor, post: torch.Tensor, @@ -224,7 +227,7 @@ def all_reduce_mhc_norm( """Plain all-reduce of ``x`` with the mHC + norm epilogue: the finalize kernel driven with identity routing (top_k = 1, unit weights).""" idx, weights = _identity_routing(x.shape[0], x.device) - return moe_finalize_all_reduce_mhc_norm( + return moe_finalize_all_reduce_mhc_post_combine_norm( x, idx, weights, @@ -254,17 +257,22 @@ def _jit_mhc_quant_module(world_size, top_k, cluster_size, weight_dtype): True, ) return load_jit( - "moe_finalize_all_reduce_mhc_quant", + "moe_finalize_all_reduce_mhc_post_combine_norm_quant", *args, cuda_files=["distributed/all_reduce_fusion.cuh"], - cuda_wrappers=[("run", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_quant")], + cuda_wrappers=[ + ( + "run", + f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_post_combine_norm_quant", + ) + ], ) @register_custom_op( mutates_args=["out", "mhc_out", "normalized", "quantized", "scales"] ) -def _moe_finalize_all_reduce_mhc_quant_op( +def _moe_finalize_all_reduce_mhc_post_combine_norm_quant_op( world_size: int, top_k: int, cluster_size: int, @@ -306,7 +314,7 @@ def _moe_finalize_all_reduce_mhc_quant_op( ) -def moe_finalize_all_reduce_mhc_quant( +def moe_finalize_all_reduce_mhc_post_combine_norm_quant( gemm2: torch.Tensor, idx: torch.Tensor, weights: torch.Tensor, @@ -322,12 +330,14 @@ def moe_finalize_all_reduce_mhc_quant( world_size: int, cluster_size: Optional[int] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """:func:`moe_finalize_all_reduce_mhc_norm` plus fp8 e4m3 quantization of - the normalized row with ue8m0 group scales (rows <= 8). Returns + """:func:`moe_finalize_all_reduce_mhc_post_combine_norm` plus fp8 e4m3 quantization of + the normalized row with ue8m0 group scales (rows <= 128: the scale swizzle + addresses a single 128-row tile). Returns ``(reduced, mhc_out, normalized, quantized, scales)``. """ rows = weights.shape[0] - assert 0 < rows <= 8 + # One 128-row scale tile; larger needs a (token/128) block offset in the kernel. + assert 0 < rows <= 128 out = torch.empty( (rows, _MHC_HIDDEN_DIM), dtype=torch.bfloat16, device=gemm2.device ) @@ -338,7 +348,7 @@ def moe_finalize_all_reduce_mhc_quant( scales = torch.empty( (_MHC_HIDDEN_DIM // 32) * 128, device=gemm2.device, dtype=torch.uint8 ) - _moe_finalize_all_reduce_mhc_quant_op( + _moe_finalize_all_reduce_mhc_post_combine_norm_quant_op( world_size, top_k, cluster_size or default_cluster_size(_MHC_HIDDEN_DIM), diff --git a/python/sglang/kernels/ops/communication/all_reduce_mhc_combine.py b/python/sglang/kernels/ops/communication/all_reduce_mhc_combine.py deleted file mode 100644 index df44106504..0000000000 --- a/python/sglang/kernels/ops/communication/all_reduce_mhc_combine.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Finalize/all-reduce + post + combine, retaining standalone RMSNorm.""" - -from typing import Optional - -import torch - -from sglang.kernels.jit.utils import ( - cache_once, - is_arch_support_pdl, - load_jit, - make_cpp_args, -) -from sglang.kernels.ops.communication.all_reduce_fusion import ( - default_cluster_size, - get_registered_comm, -) -from sglang.srt.utils.custom_op import register_custom_op - - -@cache_once -def _module(world_size, top_k, cluster_size, weight_dtype): - args = make_cpp_args( - world_size, - 5120, - top_k, - cluster_size, - is_arch_support_pdl(), - weight_dtype, - True, - False, - True, - ) - return load_jit( - "moe_finalize_all_reduce_mhc_combine", - *args, - cuda_files=["distributed/all_reduce_fusion.cuh"], - cuda_wrappers=[("run", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_combine")], - ) - - -@register_custom_op(mutates_args=["out", "mhc_out", "combined"]) -def _moe_finalize_all_reduce_mhc_combine( - world_size: int, - top_k: int, - cluster_size: int, - out: torch.Tensor, - mhc_out: torch.Tensor, - combined: torch.Tensor, - gemm2: torch.Tensor, - idx: torch.Tensor, - weights: torch.Tensor, - shared: Optional[torch.Tensor], - residual: torch.Tensor, - post: torch.Tensor, - comb: torch.Tensor, - pre: torch.Tensor, -) -> None: - comm = get_registered_comm(world_size) - assert comm is not None - _module(world_size, top_k, cluster_size, weights.dtype).run( - comm, - out, - gemm2, - idx, - weights, - shared, - mhc_out, - residual, - post, - comb, - pre, - combined, - ) - - -def moe_finalize_all_reduce_mhc_combine( - gemm2, - idx, - weights, - top_k, - shared, - residual, - post, - comb, - pre, - *, - world_size, - cluster_size=None, -): - out = torch.empty( - (weights.shape[0], 5120), dtype=torch.bfloat16, device=gemm2.device - ) - mhc_out = torch.empty_like(residual) - combined = torch.empty_like(out) - if weights.shape[0]: - _moe_finalize_all_reduce_mhc_combine( - world_size, - top_k, - cluster_size or default_cluster_size(5120), - out, - mhc_out, - combined, - gemm2, - idx, - weights, - shared, - residual, - post, - comb, - pre, - ) - return out, mhc_out, combined - - -def all_reduce_mhc_combine(x, residual, post, comb, pre, *, world_size): - from sglang.kernels.ops.communication.all_reduce_mhc import _identity_routing - - idx, weights = _identity_routing(x.shape[0], x.device) - return moe_finalize_all_reduce_mhc_combine( - x, idx, weights, 1, None, residual, post, comb, pre, world_size=world_size - ) diff --git a/python/sglang/kernels/ops/layernorm/hc_combine_norm.py b/python/sglang/kernels/ops/layernorm/hc_combine_norm.py index a52c3e17a8..b18e87260e 100644 --- a/python/sglang/kernels/ops/layernorm/hc_combine_norm.py +++ b/python/sglang/kernels/ops/layernorm/hc_combine_norm.py @@ -55,9 +55,12 @@ def _hc_combine_norm_prefill( def hc_combine_norm( x: torch.Tensor, pre: torch.Tensor, weight: torch.Tensor, eps: float ) -> torch.Tensor: - """Fuse four-stream combine and RMSNorm for BF16 batches of width 5120.""" + """Fuse four-stream combine and RMSNorm for BF16 batches of width 5120. + + Every row count takes the same per-row reduction in the same order, so the + result is batch-invariant across the grid-split variants below.""" m = x.shape[0] - assert (0 < m <= 96 or 4096 <= m <= 65536) and x.shape == (m, 20480) + assert x.shape == (m, 20480) assert pre.shape == (m, 4) and pre.stride(1) == 1 assert weight.shape == (5120,) and weight.is_contiguous() assert x.dtype == weight.dtype == torch.bfloat16 and x.stride(1) == 1 @@ -133,7 +136,7 @@ def hc_combine_norm_mxfp8( ): """Four-stream combine + RMSNorm returning ``(y_bf16, y_q, y_sf)``.""" m = x.shape[0] - assert 0 < m <= 8, "the fused MXFP8 epilogue only supports small decode/verify" + assert m <= 128, "the MXFP8 scale swizzle addresses a single 128-row tile" k = x.shape[1] // 4 y = torch.empty((m, k), dtype=x.dtype, device=x.device) q, s = _alloc(m, k, x.device) diff --git a/python/sglang/kernels/ops/layernorm/mhc_post_combine.py b/python/sglang/kernels/ops/layernorm/mhc_post_combine.py deleted file mode 100644 index 4509d6e36a..0000000000 --- a/python/sglang/kernels/ops/layernorm/mhc_post_combine.py +++ /dev/null @@ -1,85 +0,0 @@ -"""HC=4 post-mix and pre-combine with the original BF16 intermediates.""" - -import torch -import triton -import triton.language as tl - - -@triton.jit -def _mhc_post_combine(X, R, P, C, A, RO, Y, H: tl.constexpr, B: tl.constexpr): - row = tl.program_id(0) - h = tl.program_id(1) * B + tl.arange(0, B) - mask = h < H - x = tl.load(X + row * H + h, mask, 0).to(tl.float32) - r0 = tl.load(R + (row * 4 + 0) * H + h, mask, 0).to(tl.float32) - r1 = tl.load(R + (row * 4 + 1) * H + h, mask, 0).to(tl.float32) - r2 = tl.load(R + (row * 4 + 2) * H + h, mask, 0).to(tl.float32) - r3 = tl.load(R + (row * 4 + 3) * H + h, mask, 0).to(tl.float32) - collapsed = tl.full((B,), 0, tl.float32) - for i in tl.static_range(4): - post = tl.load(P + row * 4 + i) - c0 = tl.load(C + row * 16 + i) - c1 = tl.load(C + row * 16 + 4 + i) - c2 = tl.load(C + row * 16 + 8 + i) - c3 = tl.load(C + row * 16 + 12 + i) - pre = tl.load(A + row * 4 + i) - # Match mhc_post_split_h's contraction order and BF16 store. - mixed = tl.fma(post, x, c0 * r0) - mixed = tl.fma(c1, r1, mixed) - mixed = tl.fma(c2, r2, mixed) - mixed = tl.fma(c3, r3, mixed) - rounded = mixed.to(tl.bfloat16) - tl.store(RO + (row * 4 + i) * H + h, rounded, mask) - # Match hc_combine's sequential FP32 accumulation, then BF16 store. - collapsed = tl.fma(pre, rounded.to(tl.float32), collapsed) - tl.store(Y + row * H + h, collapsed, mask) - - -def mhc_post_combine(x, residual, post, comb, pre): - """Return (updated HC streams, collapsed BF16 input) out of place.""" - assert x.dtype == residual.dtype == torch.bfloat16 - assert post.dtype == comb.dtype == pre.dtype == torch.float32 - assert x.ndim == 2 and x.shape[1] == 5120 - assert residual.shape == (x.shape[0], 4, x.shape[1]) - assert post.shape == pre.shape == (x.shape[0], 4) - assert comb.shape == (x.shape[0], 4, 4) - assert all(t.is_contiguous() for t in (x, residual, post, comb, pre)) - updated = torch.empty_like(residual) - combined = torch.empty_like(x) - if x.shape[0]: - block = 1024 if x.shape[0] <= 192 or x.shape[0] >= 4096 else 512 - _mhc_post_combine[(x.shape[0], triton.cdiv(x.shape[1], block))]( - x, - residual, - post, - comb, - pre, - updated, - combined, - H=x.shape[1], - B=block, - num_warps=4, - enable_fp_fusion=False, - ) - return updated, combined - - -@triton.jit -def _hc_norm_prefill(X, W, Y, EPS: tl.constexpr): - row = tl.program_id(0).to(tl.int64) - h = tl.arange(0, 8192) - value = tl.load(X + row * 5120 + h, h < 5120, 0).to(tl.float32) - inv_rms = tl.rsqrt(tl.sum(value * value, 0) / 5120 + EPS) - weight = tl.load(W + h, h < 5120, 0).to(tl.float32) - tl.store(Y + row * 5120 + h, value * inv_rms * weight, h < 5120) - - -def hc_norm_prefill(combined, weight, eps): - """Keep the original fused prefill norm's arithmetic on collapsed input.""" - assert 4096 <= combined.shape[0] <= 65536 and combined.shape[1] == 5120 - assert weight.shape == (5120,) - assert combined.dtype == weight.dtype == torch.bfloat16 - assert combined.is_contiguous() and weight.is_contiguous() - output = torch.empty_like(combined) - _hc_norm_prefill[(combined.shape[0],)](combined, weight, output, eps, num_warps=4) - return output diff --git a/python/sglang/srt/layers/moe/mhc_post_fusion.py b/python/sglang/srt/layers/moe/mhc_post_fusion.py deleted file mode 100644 index 3d3ff204c3..0000000000 --- a/python/sglang/srt/layers/moe/mhc_post_fusion.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Scoped handoff of a decoder's HC post operands to deferred MoE finalize.""" - -from contextlib import contextmanager -from contextvars import ContextVar -from dataclasses import dataclass -from typing import Callable, Optional - -import torch - - -@dataclass -class MhcPostFusion: - residual: torch.Tensor - post: Optional[torch.Tensor] - comb: Optional[torch.Tensor] - stats_stream: Optional[torch.cuda.Stream] - output: Optional[torch.Tensor] = None - pre: Optional[torch.Tensor] = None - norm_weight: Optional[torch.Tensor] = None - norm_eps: float = 0.0 - normalized: Optional[torch.Tensor] = None - quantized: Optional[tuple[torch.Tensor, torch.Tensor]] = None - combine_only: bool = False - combined: Optional[torch.Tensor] = None - overlap_only: bool = False - record_stats: Optional[ - Callable[[], tuple[torch.Tensor, torch.Tensor, torch.Tensor]] - ] = None - - def start_stats_before_all_reduce(self): - if self.overlap_only and self.record_stats is not None: - assert self.stats_stream is not None - self.stats_stream.wait_stream(torch.cuda.current_stream()) - self.materialize_stats() - - def materialize_stats(self): - # Record after the main parent; graph replay must keep the join on the - # main stream. - if self.record_stats is not None: - self.pre, self.post, self.comb = self.record_stats() - self.record_stats = None - - -_current: ContextVar[Optional[MhcPostFusion]] = ContextVar("moe_mhc_post", default=None) - - -def current_mhc_post_fusion(): - return _current.get() - - -@contextmanager -def use_mhc_post_fusion(state): - token = _current.set(state) - try: - yield - finally: - _current.reset(token) diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py index 46f5666cdf..12f2c0ecd9 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py @@ -81,7 +81,7 @@ def is_deferred_finalize_enabled() -> bool: def finalize_flashinfer_trtllm_deferred_output( deferred_output: FlashInferTrtllmDeferredFinalizeOutput, - shared_output: torch.Tensor, + shared_output: Optional[torch.Tensor], ) -> torch.Tensor: from sglang.kernels.ops.moe.moe_finalize_fuse_shared import moe_finalize_fuse_shared diff --git a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py index 7e5f844b7d..440e16cbcb 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py @@ -519,82 +519,43 @@ def maybe_fuse_routed_scale_and_shared_add( return routed -# Fused finalize + shared add + TP all-reduce -_fused_finalize_all_reduce_world_size: Optional[int] = None -_fused_finalize_all_reduce_probed = False -_fused_finalize_all_reduce_comm = None +# Fused finalize + shared add + TP all-reduce, staged through the TP group's own +# CustomAllReduceV2 push plane (tiny batches only; no dedicated workspace). -def _fused_finalize_all_reduce_comm_world_size() -> Optional[int]: - """Reserve a separate push plane for up to 384 rows of fused MoE output.""" - global _fused_finalize_all_reduce_world_size, _fused_finalize_all_reduce_probed - global _fused_finalize_all_reduce_comm - if not _fused_finalize_all_reduce_probed: - # The eager warmup must initialize peer workspaces before capture. - if torch.cuda.is_current_stream_capturing(): - return None - _fused_finalize_all_reduce_probed = True - from sglang.kernels.ops.communication import all_reduce_fusion - from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( - CustomAllReduceV2, +def register_fused_all_reduce_comm() -> None: + """Hand the TP group's push plane to the fusion kernels, once per process. + Must run before graph capture; unregistered means everything runs unfused.""" + from sglang.kernels.ops.communication import all_reduce_fusion + from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( + CustomAllReduceV2, + ) + + ca_comm = get_parallel().tp_group.ca_comm + if isinstance(ca_comm, CustomAllReduceV2) and not ca_comm.disabled: + all_reduce_fusion.register_comm(ca_comm.obj) + else: + log_info_on_rank0( + logger, + "Fused MoE finalize: TP group has no " + "CustomAllReduceV2 push plane; keeping the unfused finalize path", ) - ca_comm = get_parallel().tp_group.ca_comm - if isinstance(ca_comm, CustomAllReduceV2) and not ca_comm.disabled: - fused_comm = ca_comm - if ca_comm.world_size == 4: - from sglang.kernels.ops.communication.mp import register_comm_cleanup - fused_comm = CustomAllReduceV2( - ca_comm.group, - ca_comm.device, - max_pull_size=0, - max_pull_blocks=0, - max_push_size=4 * 1024 * 1024, - max_push_blocks=512, - ) - register_comm_cleanup(fused_comm) - if fused_comm.disabled: - return None - _fused_finalize_all_reduce_comm = fused_comm - all_reduce_fusion.register_comm(fused_comm.obj) - _fused_finalize_all_reduce_world_size = fused_comm.world_size - else: - log_info_on_rank0( - logger, - "Fused MoE finalize: TP group has no " - "CustomAllReduceV2 push plane; keeping the unfused finalize path", - ) - return _fused_finalize_all_reduce_world_size +def can_fuse_all_reduce(num_tokens: int, hidden_dim: int) -> bool: + """Whether the registered push plane can stage ``[num_tokens, hidden_dim]`` + BF16 rows: one slot holds them and each row owns a push phase counter. The + 384 ceiling is the bench-validated range of the fused collective family. - -def should_use_fuse_finalize_all_reduce( - experts, num_tokens: int, hidden_dim: int -) -> bool: - """Capability only; the batch-size policy cap lives at the call site. The - kernel never rescales, so the expert weights must carry the routed scaling.""" - if not isinstance(experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod): - return False - if experts.quant_method.flashinfer_mxfp4_moe_precision != "default": - return False - if not experts.should_fuse_routed_scaling_factor_in_topk: - return False - if num_tokens <= 0: - return False - if num_tokens > 96: - from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled - - if is_batch_invariant_mode_enabled(): - return False + The deferred-finalize handle type carries its own numeric contract (every + producer folds the routed scaling; no consumer rescales).""" from sglang.kernels.ops.communication import all_reduce_fusion - if not all_reduce_fusion.valid_cluster_sizes(hidden_dim): + if all_reduce_fusion.get_registered_comm(get_parallel().tp_size) is None: return False - tp_group = get_parallel().tp_group - if _fused_finalize_all_reduce_comm_world_size() != tp_group.world_size: - return False - comm = _fused_finalize_all_reduce_comm - # Each token row owns a push phase counter on the epilogue's plane. - if num_tokens > comm.config.num_push_blocks: - return False - return all_reduce_fusion.fits_push_slot(comm.max_push_size, num_tokens, hidden_dim) + comm = get_parallel().tp_group.ca_comm + return ( + num_tokens <= 384 + and num_tokens <= comm.config.num_push_blocks + and all_reduce_fusion.fits_push_slot(comm.max_push_size, num_tokens, hidden_dim) + ) diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_moe.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_moe.py index 09846c3c6f..46bc91d883 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_moe.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_moe.py @@ -7,13 +7,7 @@ from typing import Optional, Tuple import torch -from sglang.kernels.ops.communication.all_reduce_mhc_hip import all_reduce_mhc_post -from sglang.srt.layers.moe.mhc_post_fusion import MhcPostFusion -from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( - ALL_REDUCE_MHC_MAX_ROWS, -) from sglang.srt.models.deepseek_common.utils import _use_aiter -from sglang.srt.runtime_context import get_parallel if _use_aiter: from sglang.kernels.ops.moe.rocm_router_gate import rocm_router_max_tokens @@ -58,27 +52,3 @@ def forward_gate( if logits_and_partials is not None: return logits_and_partials return moe.gate(hidden_states, gemm_output_zero_allocator), None - - -def fused_all_reduce_mhc( - moe, mhc: Optional[MhcPostFusion], hidden_states: torch.Tensor -) -> bool: - """Whether the post-experts all-reduce of moe (a DeepseekV2MoE) ran fused with the - mHC post, for the eagerly built 1-8 row mHC states; the decoder then reads mhc.output - and hidden_states only keeps the DP wrapper's tensor contract.""" - if not ( - mhc is not None - and not moe._shared_expert_tp1 - and not mhc.overlap_only - and mhc.post is not None - and 1 <= hidden_states.shape[0] <= ALL_REDUCE_MHC_MAX_ROWS - ): - return False - mhc.output = all_reduce_mhc_post( - hidden_states, - mhc.residual, - mhc.post, - mhc.comb, - get_parallel().tp_group.ca_comm, - ) - return True diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py index 91b3c1115d..4b84840dbd 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py @@ -1,5 +1,5 @@ import logging -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple import torch import triton @@ -10,7 +10,9 @@ if TYPE_CHECKING: from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled from sglang.srt.environ import envs from sglang.srt.layers.moe import get_moe_a2a_backend -from sglang.srt.layers.moe.mhc_post_fusion import MhcPostFusion, use_mhc_post_fusion +from sglang.srt.layers.moe.utils import should_skip_post_experts_all_reduce +from sglang.srt.models.deepseek_v2 import MoEOutput +from sglang.srt.models.deepseek_v4_mhc import AttnOutput from sglang.srt.runtime_context import ( get_exec, get_forward, @@ -38,6 +40,19 @@ if _is_gfx95_supported: post_attention_norm, ) + +class MhcPostOperands(NamedTuple): + """What the fused all-reduce + hc_post kernel reads besides the sublayer rows. + + The caller hands this to `apply_attention_mhc` / `apply_moe_mhc` directly; the + sublayer only has to be told to leave its reduce alone. + """ + + residual: torch.Tensor + post: torch.Tensor + comb: torch.Tensor + + # the fused all-reduce + hc_post kernel: DeepSeek-V4.1's hidden size, up to this many rows MHC_HIDDEN_SIZE = 5120 ALL_REDUCE_MHC_MAX_ROWS = 8 @@ -422,7 +437,7 @@ def apply_mhc_post_pre_boundary( if not _is_fused_mhc_post_pre_enabled(): return None - from sglang.srt.models.deepseek_v4 import _get_mhc_ops + from sglang.srt.models.deepseek_v4 import get_mhc_ops post_in = post.unsqueeze(-1) if post.ndim == 2 else post ( @@ -430,7 +445,7 @@ def apply_mhc_post_pre_boundary( post_out, comb_out, layer_input_out, - ) = _get_mhc_ops().mhc_fused_post_pre( + ) = get_mhc_ops().mhc_fused_post_pre( layer_input, residual, post_in, @@ -508,14 +523,12 @@ def _can_fuse_mhc(layer, residual: torch.Tensor, forward_batch) -> bool: def _make_mhc_fusion( residual: torch.Tensor, coefficients: "HcCoefficients", comm -) -> Optional[MhcPostFusion]: +) -> Optional[MhcPostOperands]: # comm is the group's aiter CustomAllreduce, or None when custom all-reduce is off if comm is None or comm.disabled or not comm.enable_register_for_capturing: return None coefficients.materialize() - return MhcPostFusion( - residual, coefficients.post, coefficients.comb, None, pre=coefficients.pre - ) + return MhcPostOperands(residual, coefficients.post, coefficients.comb) def attention_mhc_fusion(layer, residual, coefficients, forward_batch): @@ -536,21 +549,29 @@ def moe_mhc_fusion(layer, residual, coefficients, forward_batch): and layer.mlp.tp_size == 4 and not layer.mlp._shared_expert_tp1 and get_moe_a2a_backend().is_none() + # A reduce some later step already owns, or one the combine performed, is + # not ours to fuse into the post. + and not should_skip_post_experts_all_reduce(is_tp_path=True) ): return None return _make_mhc_fusion(residual, coefficients, get_parallel().tp_group.ca_comm) -def apply_attention_mhc(x: torch.Tensor, state: MhcPostFusion) -> None: - # a lazily recorded state still needs its stats before the fused kernel reads them - state.materialize_stats() - if state.stats_stream is not None: - torch.cuda.current_stream().wait_stream(state.stats_stream) - state.output = all_reduce_mhc_post( +def apply_attention_mhc(x: torch.Tensor, state: MhcPostOperands) -> torch.Tensor: + """Attention's all-reduce fused with hc_post; returns the rebuilt streams.""" + return all_reduce_mhc_post( x, state.residual, state.post, state.comb, get_parallel().attn_tp_group.ca_comm ) +def apply_moe_mhc(x: torch.Tensor, state: MhcPostOperands) -> torch.Tensor: + """The same for the MoE's post-experts reduce, on rows it merged but left + unreduced.""" + return all_reduce_mhc_post( + x, state.residual, state.post, state.comb, get_parallel().tp_group.ca_comm + ) + + def forward_hc_pre_from_prev_fused_boundary( layer, positions: torch.Tensor, @@ -581,19 +602,25 @@ def forward_hc_pre_from_prev_fused_boundary( ) with layer.self_attn.maybe_use_decode_attn_tp(forward_batch): mhc = attention_mhc_fusion(layer, residual, attn_coefficients, forward_batch) - with use_mhc_post_fusion(mhc): - x = layer.self_attn( - x=x, positions=positions, forward_batch=forward_batch, x_quant=x_quant - ) + x = layer.self_attn( + x=x, + positions=positions, + forward_batch=forward_batch, + x_quant=x_quant, + defer_all_reduce=mhc is not None, + ) if mhc is not None: - residual = mhc.output + # wo_b honors the deferred reduce under attention_mhc_fusion's gates; + # fail loud rather than fall back to an unfused reduce. + assert isinstance(x, AttnOutput) + residual = apply_attention_mhc(x.partial, mhc) x = None residual, x, ffn_coefficients = hc_boundary( layer, x, residual, - attn_coefficients.post if mhc is None else None, - attn_coefficients.comb if mhc is None else None, + attn_coefficients.post if x is not None else None, + attn_coefficients.comb if x is not None else None, attn_coefficients.pre, layer.hc_ffn_fn, layer.hc_ffn_scale, @@ -601,15 +628,22 @@ def forward_hc_pre_from_prev_fused_boundary( ) x = _gfx95_dense_post_attention_norm(layer, x, ffn_coefficients) mhc = moe_mhc_fusion(layer, residual, ffn_coefficients, forward_batch) - with use_mhc_post_fusion(mhc): - x = layer._run_moe_ffn_dp_sync( - x, forward_batch, input_ids=input_ids, input_ids_global=input_ids_global - ) + x = layer._run_moe_ffn_dp_sync( + x, + forward_batch, + input_ids=input_ids, + input_ids_global=input_ids_global, + return_moe_output=mhc is not None, + ) ffn_pre, ffn_post, ffn_comb = ffn_coefficients.tensors() - if mhc is not None and mhc.output is not None: + if mhc is not None: + # The MoE TP path honors the deferred output under moe_mhc_fusion's gates; + # fail loud rather than fall back to an unfused reduce. The fused post + # carries the reduce, so take the merged rows without one. + assert isinstance(x, MoEOutput) # Reduction already applied post. The next boundary consumes this # materialized residual, including when it would normally defer post. - return mhc.output, ffn_pre, None + return apply_moe_mhc(x.get_merged(), mhc), ffn_pre, None if defer_post: return None, ffn_pre, (x, residual, ffn_post, ffn_comb) return layer.hc_post(x, residual, ffn_post, ffn_comb), ffn_pre, None diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py index 864f0f9c00..51f47d3075 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py @@ -167,20 +167,15 @@ def init_decoder_layer(layer, quant_config) -> None: ) -def hc_post(layer, x, residual, post, comb) -> Optional[torch.Tensor]: +def hc_post(cfg, x, residual, post, comb) -> Optional[torch.Tensor]: """V4.1's gfx950 hc_post at 768-4096 rows: the split-H kernel with 2048-wide blocks, whose ordinary stores preserve locality for the following mHC reader; None elsewhere.""" if not ( _is_gfx95_supported - and layer.config.model_type == "deepseek_v41" + and cfg.pre_from_prev + and cfg.mult == 4 and 768 <= x.shape[0] <= 4096 and x.shape[1] == 5120 - and residual.shape == (x.shape[0], 4, 5120) - and post.shape == (x.shape[0], 4) - and comb.shape == (x.shape[0], 4, 4) - and x.dtype == residual.dtype == torch.bfloat16 - and post.dtype == comb.dtype == torch.float32 - and all(t.is_contiguous() for t in (x, residual, post, comb)) ): return None return mhc_post_split_h(x, residual, post, comb, block_size=2048) @@ -216,14 +211,12 @@ def input_norm( def forward_layer_fused_boundary( model, i: int, + state, *, positions: torch.Tensor, - hidden_states: Optional[torch.Tensor], input_ids: torch.Tensor, forward_batch, input_ids_global: torch.Tensor, - prev_pre: Optional[torch.Tensor], - pending_post: Optional[Tuple[torch.Tensor, ...]], capture_dspark: bool, ): """Layer i through the fused mHC boundary. Its FFN hc_post stays pending for the next @@ -235,14 +228,23 @@ def forward_layer_fused_boundary( and model.layers[nxt].engram is None and not (capture_dspark and nxt in model.dspark_layers_to_capture) ) - return forward_hc_pre_from_prev_fused_boundary( + # The boundary speaks its own (hidden_states, prev_pre, pending_post) triple; + # the loop speaks HcState. This is the only place the two meet. + from sglang.srt.models.deepseek_v4_mhc import HcPending, HcState + + pending = isinstance(state.streams, HcPending) + hidden_states, prev_pre, pending_post = forward_hc_pre_from_prev_fused_boundary( model.layers[i], positions=positions, - hidden_states=hidden_states, + hidden_states=None if pending else state.streams, input_ids=input_ids, forward_batch=forward_batch, input_ids_global=input_ids_global, - prev_pre=prev_pre, - pending_post=pending_post, + prev_pre=state.pre, + pending_post=tuple(state.streams) if pending else None, defer_post=defer_post, ) + return HcState( + hidden_states if pending_post is None else HcPending(*pending_post), + prev_pre, + ) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 737037f2b8..de12799843 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -23,7 +23,17 @@ from __future__ import annotations import logging from contextlib import contextmanager, nullcontext from functools import cached_property -from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + NamedTuple, + Optional, + Tuple, + Union, +) import torch import torch.nn.functional as F @@ -96,7 +106,6 @@ from sglang.srt.layers.moe import ( get_moe_a2a_backend, get_moe_runner_backend, post_experts_all_reduce, - should_skip_post_experts_all_reduce, should_use_flashinfer_cutlass_moe_fp4_allgather, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class @@ -133,7 +142,6 @@ from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( Mxfp8RoutedInputPreQuant, maybe_fuse_routed_scale_and_shared_add, routed_hidden_size, - should_use_fuse_finalize_all_reduce, ) from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope_wrapper @@ -582,9 +590,33 @@ class MoEGate(nn.Module): return logits -# The dedicated 4 MiB push slot fits 384 rows of 5120 BF16 values. -# The dispatch gate also checks slot capacity and available counters. -_FUSED_FINALIZE_ALL_REDUCE_MAX_TOKENS = 384 +class MoEOutput(NamedTuple): + """A MoE block's pieces, before anything merges or reduces them. ``routed`` is + a deferred finalize handle or the raw routed rows.""" + + routed: Any + shared: Optional[torch.Tensor] + experts: nn.Module + routed_scaling_factor: float + shared_is_replicated: bool + + def get_merged(self) -> torch.Tensor: + """Merge the pieces, leaving the reduction to the caller.""" + from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( + FlashInferTrtllmDeferredFinalizeOutput, + finalize_flashinfer_trtllm_deferred_output, + ) + + # NOTE: normal TP-sharded shared expert out + shared = None if self.shared_is_replicated else self.shared + if isinstance(self.routed, FlashInferTrtllmDeferredFinalizeOutput): + return finalize_flashinfer_trtllm_deferred_output(self.routed, shared) + return maybe_fuse_routed_scale_and_shared_add( + self.experts, + self.routed, + shared, + self.routed_scaling_factor, + ) class DeepseekV2MoE(nn.Module): @@ -645,13 +677,6 @@ class DeepseekV2MoE(nn.Module): self.routed_quant_stream = routed_quant_stream self.is_nextn = is_nextn self.is_deepseek_v4 = is_deepseek_v4 - self._fuse_finalize_all_reduce = ( - is_deepseek_v4 - and getattr(config, "hc_pre_from_prev_sublayer", False) - and get_platform().is_blackwell - and self.tp_size == 4 - ) - n_hash_layers = getattr(config, "num_hash_layers", 0) self.is_hash = layer_id < n_hash_layers and not (is_deepseek_v4 and is_nextn) @@ -957,7 +982,11 @@ class DeepseekV2MoE(nn.Module): input_ids: Optional[torch.Tensor] = None, input_ids_global: Optional[torch.Tensor] = None, skip_shared_experts: bool = False, - ) -> torch.Tensor: + return_moe_output: bool = False, + ) -> Union[torch.Tensor, MoEOutput]: + """``return_moe_output`` asks for the pieces unmerged, as `MoEOutput`; only + the TP paths honor it. Distinct from `get_forward().defer_moe_finalize`, + the V2/V3 layer-boundary handoff.""" from sglang.srt.layers.moe.mega_moe import forward_mega_moe, should_use_mega_moe if should_use_mega_moe(self, hidden_states): @@ -998,6 +1027,7 @@ class DeepseekV2MoE(nn.Module): input_ids_global=input_ids_global, num_token_non_padded=num_token_non_padded, use_vision_topk=use_vision_topk, + return_moe_output=return_moe_output, ) else: return self.forward_normal( @@ -1008,6 +1038,7 @@ class DeepseekV2MoE(nn.Module): skip_shared_experts=skip_shared_experts, num_token_non_padded=num_token_non_padded, use_vision_topk=use_vision_topk, + return_moe_output=return_moe_output, ) else: return self.forward_deepep( @@ -1022,7 +1053,8 @@ class DeepseekV2MoE(nn.Module): input_ids_global: Optional[torch.Tensor] = None, num_token_non_padded: Optional[torch.Tensor] = None, use_vision_topk: bool = False, - ) -> torch.Tensor: + return_moe_output: bool = False, + ) -> Union[torch.Tensor, MoEOutput]: # Note(kpham-sgl): issue order satisfies 3 constraints: # - no stream explosion: main (routed) issued before alt block -> capture reuses 1 alt stream; # - PDL overlap: routed is the last main-stream kernel (fuses w/ residual add); @@ -1103,22 +1135,7 @@ class DeepseekV2MoE(nn.Module): ) ready = self.routed_quant_stream.record_event() routed_pre_quant_input = Mxfp8RoutedInputPreQuant(x_q, x_sf, ready) - # The mHC post-split consumes the reduced row without an RMSNorm. - use_fused_finalize_all_reduce = ( - self._fuse_finalize_all_reduce - # The kernel completes the tensor-parallel sum itself. - and self.reduce_results - and has_shared_output - and hidden_states.shape[-1] == 5120 - and not self._shared_expert_tp1 - and self.tp_size > 1 - and hidden_states.shape[0] <= _FUSED_FINALIZE_ALL_REDUCE_MAX_TOKENS - and not should_skip_post_experts_all_reduce(is_tp_path=True) - and should_use_fuse_finalize_all_reduce( - self.experts, hidden_states.shape[0], hidden_states.shape[-1] - ) - ) - deferred_finalize = use_fused_finalize_all_reduce or ( + deferred_finalize = ( has_shared_output and not self._shared_expert_tp1 and topk_output.format == TopKOutputFormat.BYPASSED @@ -1128,6 +1145,11 @@ class DeepseekV2MoE(nn.Module): or hidden_states.shape[0] <= self._deferred_finalize_max_tokens ) ) + # NOTE: if `return_moe_output`, always try to defer finalize + deferred_finalize = deferred_finalize or ( + return_moe_output + and isinstance(self.experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod) + ) if deferred_finalize: final_hidden_states = self.experts.forward_deferred_finalize( hidden_states, topk_output, pre_quant_input=routed_pre_quant_input @@ -1174,91 +1196,24 @@ class DeepseekV2MoE(nn.Module): reduce=sum_post_experts_output, ) - all_reduce_done = False + if return_moe_output: + return MoEOutput( + routed=final_hidden_states, + shared=shared_output, + experts=self.experts, + routed_scaling_factor=self.routed_scaling_factor, + shared_is_replicated=self._shared_expert_tp1, + ) + if deferred_finalize: from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( finalize_flashinfer_trtllm_deferred_output, ) - deferred = final_hidden_states - if ( - use_fused_finalize_all_reduce - and deferred.gemm2_out.shape[1] == hidden_states.shape[-1] - ): - from sglang.kernels.ops.communication.all_reduce_fusion import ( - moe_finalize_all_reduce, - ) - from sglang.srt.layers.moe.mhc_post_fusion import ( - current_mhc_post_fusion, - ) - - mhc = current_mhc_post_fusion() - if mhc is not None: - from sglang.kernels.ops.communication.all_reduce_mhc import ( - moe_finalize_all_reduce_mhc, - ) - - # Join the coefficients before the fused epilogue reads them. - mhc.materialize_stats() - if mhc.stats_stream is not None: - current_stream.wait_stream(mhc.stats_stream) - args = ( - deferred.gemm2_out, - deferred.expanded_idx_to_permuted_idx, - deferred.expert_weights, - deferred.top_k, - shared_output, - mhc.residual, - mhc.post, - mhc.comb, - ) - if mhc.combine_only: - from sglang.kernels.ops.communication.all_reduce_mhc_combine import ( - moe_finalize_all_reduce_mhc_combine, - ) - - final_hidden_states, mhc.output, mhc.combined = ( - moe_finalize_all_reduce_mhc_combine( - *args, mhc.pre, world_size=self.tp_size - ) - ) - elif mhc.norm_weight is not None: - from sglang.kernels.ops.communication.all_reduce_mhc import ( - moe_finalize_all_reduce_mhc_quant, - ) - - final_hidden_states, mhc.output, mhc.normalized, q, sf = ( - moe_finalize_all_reduce_mhc_quant( - *args, - mhc.pre, - mhc.norm_weight, - mhc.norm_eps, - world_size=self.tp_size, - ) - ) - mhc.quantized = (q, sf) - else: - final_hidden_states, mhc.output = moe_finalize_all_reduce_mhc( - *args, world_size=self.tp_size - ) - else: - final_hidden_states = moe_finalize_all_reduce( - deferred.gemm2_out, - deferred.expanded_idx_to_permuted_idx, - deferred.expert_weights, - deferred.top_k, - shared_output, - world_size=self.tp_size, - hidden_dim=hidden_states.shape[-1], - # Routing metadata must be ready before it is consumed. - prefetch_metadata=False, - ) - all_reduce_done = True - else: - final_hidden_states = finalize_flashinfer_trtllm_deferred_output( - deferred, - shared_output, - ) + final_hidden_states = finalize_flashinfer_trtllm_deferred_output( + final_hidden_states, + shared_output, + ) else: final_hidden_states = maybe_fuse_routed_scale_and_shared_add( self.experts, @@ -1266,24 +1221,7 @@ class DeepseekV2MoE(nn.Module): None if self._shared_expert_tp1 else shared_output, self.routed_scaling_factor, ) - - if not all_reduce_done and self.reduce_results: - if ( - self.is_deepseek_v4 - and self.tp_size > 1 - and not should_skip_post_experts_all_reduce(is_tp_path=True) - ): - from sglang.srt.layers.moe.mhc_post_fusion import ( - current_mhc_post_fusion, - ) - - mhc = current_mhc_post_fusion() - if _is_hip and _hip_moe.fused_all_reduce_mhc( - self, mhc, final_hidden_states - ): - return final_hidden_states - if mhc is not None: - mhc.start_stats_before_all_reduce() + if self.reduce_results: final_hidden_states = post_experts_all_reduce(final_hidden_states) # TP1 shared experts are replicated: add them after this block's # all-reduce, or on TP rank 0 only when the stage boundary sums. @@ -1300,7 +1238,8 @@ class DeepseekV2MoE(nn.Module): skip_shared_experts: bool = False, num_token_non_padded: Optional[torch.Tensor] = None, use_vision_topk: bool = False, - ) -> torch.Tensor: + return_moe_output: bool = False, + ) -> Union[torch.Tensor, MoEOutput]: if hasattr(self, "shared_experts") and use_intel_amx_backend( self.shared_experts.gate_up_proj ): @@ -1435,30 +1374,22 @@ class DeepseekV2MoE(nn.Module): pre_quant_input=pre_quant_input, ) + if return_moe_output: + return MoEOutput( + routed=final_hidden_states, + shared=shared_output, + experts=self.experts, + routed_scaling_factor=self.routed_scaling_factor, + shared_is_replicated=self._shared_expert_tp1, + ) + final_hidden_states = maybe_fuse_routed_scale_and_shared_add( self.experts, final_hidden_states, None if self._shared_expert_tp1 else shared_output, self.routed_scaling_factor, ) - if self.reduce_results: - if ( - self.is_deepseek_v4 - and self.tp_size > 1 - and not should_skip_post_experts_all_reduce(is_tp_path=True) - ): - from sglang.srt.layers.moe.mhc_post_fusion import ( - current_mhc_post_fusion, - ) - - mhc = current_mhc_post_fusion() - if _is_hip and _hip_moe.fused_all_reduce_mhc( - self, mhc, final_hidden_states - ): - return final_hidden_states - if mhc is not None: - mhc.start_stats_before_all_reduce() final_hidden_states = post_experts_all_reduce(final_hidden_states) # TP1 shared experts are replicated: add them after this block's # all-reduce, or on TP rank 0 only when the stage boundary sums. diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 270909408b..fda4135561 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -39,7 +39,6 @@ from sglang.kernels.ops.gemm.dsv4_wo_a import ( wo_a_bf16_small_batch, wo_a_bf16_small_batch_mxfp8, ) -from sglang.kernels.ops.layernorm.mhc_post_split_h import mhc_post_split_h from sglang.kernels.ops.quantization.fp8_kernel import ( sglang_per_token_group_quant_fp8, ) @@ -57,7 +56,6 @@ from sglang.srt.hardware_backend.npu.dsv4.dsv4_rope import ( rope_cos_sin, ) from sglang.srt.hardware_backend.npu.utils import ( - is_npu_arch35, use_npu_arch35_mxfp8_wo_a, ) from sglang.srt.layers.attention.dsa.utils import ( @@ -107,6 +105,7 @@ from sglang.srt.layers.moe import get_moe_a2a_backend, should_use_dp_reduce_scat from sglang.srt.layers.moe.fused_moe_triton import FusedMoE from sglang.srt.layers.moe.utils import ( is_shared_experts_fusion_disabled, + should_skip_post_experts_all_reduce, uses_per_rank_fused_shared_slots, ) from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod @@ -114,6 +113,9 @@ from sglang.srt.layers.quantization.fp8_utils import ( Mxfp8DenseGemmBackend, view_aiter_fused_rms_transposed_fp8_scale, ) +from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( + can_fuse_all_reduce, +) from sglang.srt.layers.quantization.mxfp8_input import Mxfp8SwizzledInput from sglang.srt.layers.rotary_embedding import get_rope_wrapper from sglang.srt.layers.utils import PPMissingLayer, get_layer_id @@ -155,6 +157,7 @@ from sglang.srt.model_loader.weight_utils import ( RUNAI_STREAMER_TENSOR_ATTR, default_weight_loader, ) +from sglang.srt.models import deepseek_v4_mhc as mhc from sglang.srt.models.dbrx import ReplicatedLinear from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( apply_mhc_post_pre_boundary, @@ -214,7 +217,7 @@ class MhcOps(NamedTuple): @functools.cache -def _get_mhc_ops() -> MhcOps: +def get_mhc_ops() -> MhcOps: """Load MHC kernels only when a DeepSeek-V4 layer needs them. Model modules are imported eagerly by the registry. Importing @@ -2148,8 +2151,9 @@ class MQALayer(MqaAttentionBase): x: torch.Tensor, positions: torch.Tensor, forward_batch: ForwardBatch, - x_quant=None, - ) -> torch.Tensor: + x_quant: Optional[torch.Tensor] = None, + defer_all_reduce: bool = False, + ) -> Union[torch.Tensor, mhc.AttnOutput]: if not get_attn_tp_context().input_scattered and x.shape[0] == 0: return x @@ -2604,56 +2608,26 @@ class MQALayer(MqaAttentionBase): self.o_lora_rank, ) - return self._project_wo_b(o) + return self._project_wo_b(o, defer_all_reduce) - def _project_wo_b(self, o): - from sglang.srt.layers.moe.mhc_post_fusion import current_mhc_post_fusion + def _project_wo_b(self, o, defer_all_reduce: bool = False): + from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce - mhc = current_mhc_post_fusion() - o, _ = self.wo_b( - o.flatten(1) if isinstance(o, torch.Tensor) else o, - skip_all_reduce=mhc is not None, + defer_all_reduce = defer_all_reduce and ( + self.attn_tp_size > 1 + and self.attn_tp_size == get_parallel().tp_size + and self.wo_b.reduce_results + and not self.wo_b.use_decode_attn_tp + and not should_skip_mlp_all_reduce() ) - if mhc is not None and mhc.overlap_only: - mhc.start_stats_before_all_reduce() - o = attn_tp_all_reduce(o) - elif mhc is not None and _is_hip: - _hip.apply_attention_mhc(o, mhc) - elif mhc is not None: - from sglang.kernels.ops.communication.all_reduce_mhc import ( - all_reduce_mhc_norm, - ) - - mhc.materialize_stats() - if mhc.stats_stream is not None: - torch.cuda.current_stream().wait_stream(mhc.stats_stream) - if mhc.combine_only: - from sglang.kernels.ops.communication.all_reduce_mhc_combine import ( - all_reduce_mhc_combine, - ) - - o, mhc.output, mhc.combined = all_reduce_mhc_combine( - o, - mhc.residual, - mhc.post, - mhc.comb, - mhc.pre, - world_size=self.attn_tp_size, - ) - else: - o, mhc.output, mhc.normalized = all_reduce_mhc_norm( - o, - mhc.residual, - mhc.post, - mhc.comb, - mhc.pre, - mhc.norm_weight, - mhc.norm_eps, - world_size=self.attn_tp_size, - ) + o, _ = self.wo_b( + o if isinstance(o, Mxfp8SwizzledInput) else o.flatten(1), + skip_all_reduce=defer_all_reduce, + ) + if defer_all_reduce: + return mhc.AttnOutput(o) if self.attn_tp_size > 1 and self.attn_tp_size < get_parallel().tp_size: o = attn_tp_all_reduce(o) - return o # ---- TBO op decomposition (prefill two-batch-overlap) ---- @@ -2771,6 +2745,23 @@ class DeepseekV4DecoderLayer(nn.Module): self.use_fused_mhc_post_pre = False if _is_hip: _hip.init_decoder_layer(self, quant_config) + self.hc_cfg = mhc.HcConfig( + mult=hc_mult, + sinkhorn_iters=config.hc_sinkhorn_iters, + eps=config.hc_eps, + rms_eps=config.rms_norm_eps, + hidden=config.hidden_size, + pre_from_prev=self.hc_pre_from_prev_sublayer, + cp_prefill=self.dsa_enable_prefill_cp, + ) + self.attn_hc: Optional[mhc.HcSubLayer] = None + self.ffn_hc: Optional[mhc.HcSubLayer] = None + self._hc_attn_tf32_parts = self._hc_ffn_tf32_parts = None + self._hc_attn_bf16_parts = self._hc_ffn_bf16_parts = None + self._init_hyper_connections() + self._next_layer: Optional[DeepseekV4DecoderLayer] = None + self.local_boundary: Optional[mhc.HcNextBoundary] = None + self.next_boundary: Optional[mhc.HcNextBoundary] = None self.engram = None if engram_layout is not None and layer_id in engram_layout.layer_ids: self.engram = Engram( @@ -2792,7 +2783,7 @@ class DeepseekV4DecoderLayer(nn.Module): prefix: str, alt_streams: Optional[List[torch.cuda.Stream]], compress_ratio_override: Optional[int], - ) -> nn.Module: + ) -> MQALayer: return MQALayer( config=config, layer_id=layer_id, @@ -2802,6 +2793,54 @@ class DeepseekV4DecoderLayer(nn.Module): compress_ratio_override=compress_ratio_override, ) + def bind_next(self, next_layer: Optional[nn.Module]) -> None: + """Save the next layer for mhc. Must be called before `_init_boundaries`""" + if not self.hc_pre_from_prev_sublayer: + return + self._next_layer = None + if isinstance(next_layer, DeepseekV4DecoderLayer) and next_layer.engram is None: + self._next_layer = next_layer + + def _init_hyper_connections(self) -> None: + """(Re)bind the two hyper-connections to the current mixing weights.""" + if not self.hc_pre_from_prev_sublayer: + return + self.attn_hc = mhc.HcSubLayer( + self.hc_cfg, + self.hc_attn_fn, + self.hc_attn_scale, + self.hc_attn_base, + self.input_layernorm, + self._hc_attn_tf32_parts, + self._hc_attn_bf16_parts, + ) + self.ffn_hc = mhc.HcSubLayer( + self.hc_cfg, + self.hc_ffn_fn, + self.hc_ffn_scale, + self.hc_ffn_base, + self.post_attention_layernorm, + self._hc_ffn_tf32_parts, + self._hc_ffn_bf16_parts, + ) + + def _init_boundaries(self) -> None: + """Resolved on first forward: `accepts_mxfp8_swizzled_input` is not settled + until `process_weights_after_loading`.""" + if self.local_boundary is not None: + return + # ATTN[i] -> MOE[i] + self.local_boundary = mhc.make_boundary( + self.post_attention_layernorm, + accepts_mxfp8=False, + ) + # MOE[i] -> ATTN[i+1] + if self._next_layer is not None: + self.next_boundary = mhc.make_boundary( + self._next_layer.input_layernorm, + accepts_mxfp8=self._next_layer.self_attn.accepts_mxfp8_swizzled_input(), + ) + def refresh_mhc_norm_weight_cache(self): # Cache bf16 norm weights so the fused path does not allocate/cast per forward. self._input_layernorm_weight_bf16 = ( @@ -2824,41 +2863,42 @@ class DeepseekV4DecoderLayer(nn.Module): and getattr(self.config, "model_type", None) == "deepseek_v41" and not is_batch_invariant_mode_enabled() ): - if get_platform().is_sm90: - from sglang.kernels.ops.layernorm.mhc import split_bf16_hc_weight + from sglang.kernels.ops.layernorm.mhc import split_bf16_hc_weight - # The compensated BF16 projection uses ordinary tensor cores; - # it does not require Blackwell or DeepGEMM's prenorm kernel. + if get_platform().is_sm90: + # Hopper's compensated projection does not require DeepGEMM. self._hc_attn_bf16_parts = split_bf16_hc_weight(self.hc_attn_fn.data) self._hc_ffn_bf16_parts = split_bf16_hc_weight(self.hc_ffn_fn.data) - return + else: + from sglang.kernels.ops.layernorm.mhc import split_tf32_hc_weight + from sglang.srt.layers.deep_gemm_wrapper.configurer import ( + ENABLE_JIT_DEEPGEMM, + ) - from sglang.kernels.ops.layernorm.mhc import ( - split_tf32_hc_weight, - ) - from sglang.srt.layers.deep_gemm_wrapper.configurer import ( - ENABLE_JIT_DEEPGEMM, - ) + if ENABLE_JIT_DEEPGEMM: + import deep_gemm - if ENABLE_JIT_DEEPGEMM: - import deep_gemm - - if not callable(getattr(deep_gemm, "tf32_hc_prenorm_gemm", None)): - return - self._hc_attn_tf32_parts = split_tf32_hc_weight(self.hc_attn_fn.data) - self._hc_ffn_tf32_parts = split_tf32_hc_weight(self.hc_ffn_fn.data) - if ( - getattr(getattr(self, "config", None), "model_type", None) - == "deepseek_v41" - ): - from sglang.kernels.ops.layernorm.mhc import ( - split_bf16_hc_weight, - ) - - self._hc_attn_bf16_parts = split_bf16_hc_weight( - self.hc_attn_fn.data - ) - self._hc_ffn_bf16_parts = split_bf16_hc_weight(self.hc_ffn_fn.data) + if callable(getattr(deep_gemm, "tf32_hc_prenorm_gemm", None)): + self._hc_attn_tf32_parts = split_tf32_hc_weight( + self.hc_attn_fn.data + ) + self._hc_ffn_tf32_parts = split_tf32_hc_weight( + self.hc_ffn_fn.data + ) + self._hc_attn_bf16_parts = split_bf16_hc_weight( + self.hc_attn_fn.data + ) + self._hc_ffn_bf16_parts = split_bf16_hc_weight( + self.hc_ffn_fn.data + ) + if self.hc_pre_from_prev_sublayer: + self._init_hyper_connections() + # The fuse gates and boundaries snapshot load-time facts; weight updates + # re-run this, so drop them and re-resolve on the next forward. + self.__dict__.pop("_can_fuse_attn_mhc", None) + self.__dict__.pop("_can_fuse_ffn_mhc", None) + self.local_boundary = None + self.next_boundary = None def _input_norm( self, @@ -2903,7 +2943,7 @@ class DeepseekV4DecoderLayer(nn.Module): shape, dtype = x.size(), x.dtype if _is_npu: - return _get_mhc_ops().npu_hc_pre( + return get_mhc_ops().npu_hc_pre( x, hc_fn, hc_scale, @@ -2929,7 +2969,7 @@ class DeepseekV4DecoderLayer(nn.Module): norm_kwargs["norm_weight"] = norm.weight.data norm_kwargs["norm_eps"] = norm.variance_epsilon - post, comb, y = _get_mhc_ops().mhc_pre( + post, comb, y = get_mhc_ops().mhc_pre( residual=x, fn=hc_fn, hc_scale=hc_scale, @@ -3017,7 +3057,7 @@ class DeepseekV4DecoderLayer(nn.Module): else: x_flat, mixes = hc_pre_torch_impl(x, hc_fn) - pre, post, comb = _get_mhc_ops().hc_split_sinkhorn( + pre, post, comb = get_mhc_ops().hc_split_sinkhorn( mixes, hc_scale, hc_base, @@ -3045,93 +3085,12 @@ class DeepseekV4DecoderLayer(nn.Module): residual: torch.Tensor, post: torch.Tensor, comb: torch.Tensor, - ): - if x.shape[0] == 0: - return torch.empty( - (0, self.hc_mult, x.shape[-1]), dtype=x.dtype, device=x.device - ) - - if _is_npu: - if not is_npu_arch35(): - return torch.ops.custom.npu_hc_post(x, residual, post, comb) - # The A5 build of npu_hc_post is batched — it requires a leading - # batch axis on every operand. - return torch.ops.custom.npu_hc_post( - x.unsqueeze(0), - residual.unsqueeze(0), - post.unsqueeze(0), - comb.unsqueeze(0), - ).squeeze(0) - - if _is_xpu: - return _get_mhc_ops().mhc_post(x, residual, post, comb) - - if ( - _is_cuda - and get_platform().is_blackwell - and self.hc_pre_from_prev_sublayer - and self.hc_mult == 4 - and x.shape[1] == 5120 - and x.shape[0] <= 384 - and x.dtype == residual.dtype == torch.bfloat16 - and post.dtype == comb.dtype == torch.float32 - and all(t.is_contiguous() for t in (x, residual, post, comb)) - ): - return mhc_post_split_h(x, residual, post, comb) - - if _is_hip: - y = _hip.hc_post(self, x, residual, post, comb) - if y is not None: - return y - - if envs.SGLANG_OPT_USE_FLASHINFER_MHC.get(): - from flashinfer.mhc import mhc_post - - return mhc_post(x, residual, post, comb) - - if envs.SGLANG_OPT_USE_TILELANG_MHC_POST.get(): - if ( - self.hc_pre_from_prev_sublayer - and get_platform().is_sm90 - and x.is_cuda - and 1 <= x.shape[0] <= 64 - and x.shape[1] == 5120 - and residual.shape == (x.shape[0], 4, 5120) - and x.dtype == residual.dtype == torch.bfloat16 - and post.dtype == comb.dtype == torch.float32 - and post.numel() == x.shape[0] * 4 - and comb.shape == (x.shape[0], 4, 4) - and all(t.is_contiguous() for t in (x, residual, post, comb)) - ): - return mhc_post_split_h(x, residual, post, comb) - - from sglang.kernels.ops.layernorm.mhc import mhc_post - - return mhc_post(x, residual, post, comb) - - elif _is_hip: - from aiter.ops.mhc import mhc_post - - result = torch.empty_like(residual) - mhc_post(result, x, residual, post, comb) - return result - - assert residual.shape == (x.shape[0], self.hc_mult, x.shape[-1]) - assert post.shape == (x.shape[0], self.hc_mult) - assert comb.shape == (x.shape[0], self.hc_mult, self.hc_mult) - - @compile_in_capture_mode - def hc_post_torch_impl(x, residual, post, comb): - return ( - post.unsqueeze(-1) * x.unsqueeze(1) - + (comb.unsqueeze(-1) * residual.unsqueeze(2)).sum(dim=1) - ).type_as(x) - - return hc_post_torch_impl(x, residual, post, comb) + ) -> torch.Tensor: + return mhc.post(self.hc_cfg, x, residual, post, comb) def forward( self, - positions: torch.tensor, + positions: torch.Tensor, hidden_states: torch.Tensor, input_ids: torch.Tensor, forward_batch: ForwardBatch, @@ -3298,590 +3257,105 @@ class DeepseekV4DecoderLayer(nn.Module): # cross-layer fusion, and the final layer is completed in DeepseekV4Model. return hidden_states, residual, post, comb - def _hc_combine( - self, - x: torch.Tensor, - apply_pre: Optional[torch.Tensor], - norm: RMSNorm, - stats_stream: Optional[torch.cuda.Stream] = None, - quantized: Optional[list] = None, - normalized: Optional[torch.Tensor] = None, - precomputed: Optional[tuple] = None, - combined: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - from sglang.kernels.ops.layernorm.mhc import hc_combine - - quantize = quantized is not None - x_flat = x.flatten(1) - tiny = 0 < x.shape[0] <= 8 - if stats_stream is not None and not tiny: - stats_stream.wait_stream(torch.cuda.current_stream()) - - def combine_and_norm(): - if precomputed is not None: - assert quantized is not None - quantized.append(precomputed[1]) - return precomputed[0] - if normalized is not None: - # Prefill projections still quantize the BF16 input themselves; - # the optional fused-quantization list stays empty for this case. - assert not quantize or 4096 <= x.shape[0] <= 65536 - return normalized - if combined is not None: - if ( - 4096 <= combined.shape[0] <= 65536 - and norm.weight.dtype == torch.bfloat16 - and not norm.cast_x_before_out_mul - and norm.variance_size_override is None - ): - from sglang.kernels.ops.layernorm.mhc_post_combine import ( - hc_norm_prefill, - ) - - return hc_norm_prefill(combined, norm.weight, norm.variance_epsilon) - return norm(combined) - if apply_pre is None: - return norm(x[:, 0, :].contiguous()) - from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled - - if ( - x.is_cuda - and ( - get_platform().is_blackwell - or (get_platform().is_sm90 and not quantize) - ) - and ( - 0 < x.shape[0] <= 96 - or ( - self.config.model_type == "deepseek_v41" - and 4096 <= x.shape[0] <= 65536 - ) - ) - and self.hc_mult == 4 - and x_flat.shape[1] == 20480 - and x.dtype == norm.weight.dtype == torch.bfloat16 - and apply_pre.stride(1) == 1 - and not norm.cast_x_before_out_mul - and norm.variance_size_override is None - and not is_batch_invariant_mode_enabled() - ): - # The fused scale writer supports the small decode/verify tile only. - if quantize and x.shape[0] <= 8: - from sglang.kernels.ops.layernorm.hc_combine_norm import ( - hc_combine_norm_mxfp8, - ) - - y, y_q, y_sf = hc_combine_norm_mxfp8( - x_flat, apply_pre, norm.weight, norm.variance_epsilon - ) - quantized.append(Mxfp8SwizzledInput(y_q, y_sf)) - return y - from sglang.kernels.ops.layernorm.hc_combine_norm import hc_combine_norm - - return hc_combine_norm( - x_flat, apply_pre, norm.weight, norm.variance_epsilon - ) - return norm(hc_combine(x_flat, apply_pre, self.hc_mult, x.dtype)) - - y = combine_and_norm() - if stats_stream is not None and tiny: - stats_stream.wait_stream(torch.cuda.current_stream()) - return y - - def _hc_mix_stats( - self, - x: torch.Tensor, - hc_fn: torch.Tensor, - hc_scale: torch.Tensor, - hc_base: torch.Tensor, - stats_stream: Optional[torch.cuda.Stream] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - if stats_stream is None: - return DeepseekV4DecoderLayer._hc_mix_stats_impl( - self, x, hc_fn, hc_scale, hc_base - ) - main_stream = torch.cuda.current_stream() - x.record_stream(stats_stream) - with torch.cuda.stream(stats_stream): - coefficients = DeepseekV4DecoderLayer._hc_mix_stats_impl( - self, x, hc_fn, hc_scale, hc_base - ) - # The producer fork is in _hc_combine; consumers join before hc_post. - for coefficient in coefficients: - coefficient.record_stream(main_stream) - return coefficients - - def _hc_mix_stats_impl( - self, - x: torch.Tensor, - hc_fn: torch.Tensor, - hc_scale: torch.Tensor, - hc_base: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - from sglang.kernels.ops.layernorm.mhc import hc_mix_stats, hc_mix_stats_sinkhorn - - x_flat = x.flatten(1) - - from sglang.srt.batch_invariant_ops import ( - is_batch_invariant_mode_enabled, - ) - - parts = bf16_parts = None - hopper_medium = get_platform().is_sm90 and 32 <= x_flat.shape[0] < 4096 - if ( - x.is_cuda - and (x_flat.shape[0] >= 128 or hopper_medium) - and x_flat.is_contiguous() - and (get_platform().is_sm100 or get_platform().is_sm90) - and envs.SGLANG_OPT_DEEPGEMM_HC_PRENORM.get() - and not is_batch_invariant_mode_enabled() - ): - if hc_fn is self.hc_attn_fn: - parts = getattr(self, "_hc_attn_tf32_parts", None) - bf16_parts = getattr(self, "_hc_attn_bf16_parts", None) - elif hc_fn is self.hc_ffn_fn: - parts = getattr(self, "_hc_ffn_tf32_parts", None) - bf16_parts = getattr(self, "_hc_ffn_bf16_parts", None) - - use_bf16_projection = bf16_parts is not None and ( - hopper_medium or 4096 <= x_flat.shape[0] <= 65536 - ) - hopper_fused_stats = get_platform().is_sm90 and ( - x.shape[0] == 1 or (bf16_parts is not None and 32 <= x.shape[0] <= 65536) - ) - # gfx950 uses the fused Triton port at every row count (MI350X). - use_fused_stats = _is_gfx95_supported or ( - torch.version.cuda is not None - and (get_platform().is_blackwell or hopper_fused_stats) - ) - if x.is_cuda and use_fused_stats and x.dtype == torch.bfloat16: - # The default split-K/Sinkhorn fusion preserves batch invariance; - # compensated projections above are disabled in batch-invariant mode. - if use_bf16_projection: - from sglang.kernels.ops.layernorm.mhc import ( - hc_mix_stats_sinkhorn_bf16x3, - ) - - pre, post, comb = hc_mix_stats_sinkhorn_bf16x3( - x_flat, - bf16_parts, - hc_scale, - hc_base, - self.hc_sinkhorn_iters, - self.rms_norm_eps, - self.hc_eps, - ) - elif parts is not None: - from sglang.kernels.ops.layernorm.mhc import ( - hc_mix_stats_sinkhorn_deepgemm, - ) - - pre, post, comb = hc_mix_stats_sinkhorn_deepgemm( - x_flat, - parts, - hc_scale, - hc_base, - self.hc_sinkhorn_iters, - self.rms_norm_eps, - self.hc_eps, - ) - else: - pre, post, comb = hc_mix_stats_sinkhorn( - x_flat, - hc_fn, - hc_scale, - hc_base, - self.hc_mult, - self.hc_sinkhorn_iters, - self.rms_norm_eps, - self.hc_eps, - ) - return pre, post, comb - if x.is_cuda and torch.version.cuda is not None: - # cuBLAS/torch reductions can change order with num_tokens; this kernel - # keeps the mixing and RMS reductions batch-invariant. - mixes = hc_mix_stats(x_flat, hc_fn, self.rms_norm_eps).unsqueeze(1) - else: - x_flat = x_flat.float() - rsqrt = torch.rsqrt( - x_flat.square().mean(-1, keepdim=True) + self.rms_norm_eps - ) - mixes = (F.linear(x_flat, hc_fn) * rsqrt).unsqueeze(1) - pre, post, comb = _get_mhc_ops().hc_split_sinkhorn( - mixes, - hc_scale, - hc_base, - self.hc_mult, - self.hc_sinkhorn_iters, - self.hc_eps, - ) - return pre.squeeze(1), post.squeeze(1), comb.squeeze(1) - - def _hc_mix_and_combine( - self, - x, - hc_fn, - hc_scale, - hc_base, - apply_pre, - norm, - stats_stream=None, - quantized=None, - normalized=None, - precomputed=None, - ): - y = DeepseekV4DecoderLayer._hc_combine( - self, x, apply_pre, norm, stats_stream, quantized, normalized, precomputed - ) + @functools.cached_property + def _can_fuse_attn_mhc(self) -> bool: + """The static half of the attention fuse gate (dsv4.1 only); load-time + facts, dropped by `refresh_mhc_norm_weight_cache`. Row capacity is the + dynamic half, checked per forward via `can_fuse_all_reduce`.""" return ( - y, - *DeepseekV4DecoderLayer._hc_mix_stats( - self, x, hc_fn, hc_scale, hc_base, stats_stream - ), + get_parallel().tp_size == self.self_attn.attn_tp_size == 4 + and self.self_attn.wo_b.reduce_results + and self.local_boundary.norm_fusable + and mhc.can_fuse_post(self.hc_cfg) ) - def _get_hc_stats_stream(self, hidden_states, forward_batch): - # Prefill stats share one model-wide stream. Start them immediately - # before the sublayer's all-reduce, after its compute has completed. - if ( - self.config.model_type == "deepseek_v41" - and hidden_states.is_cuda - and get_platform().is_blackwell - and forward_batch.forward_mode.is_extend_without_speculative() - and 4096 <= hidden_states.shape[0] <= 65536 - and get_parallel().attn_dp_size == 1 - and not get_forward().sp_active - and not self.dsa_enable_prefill_cp - ): - from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled - - if not is_batch_invariant_mode_enabled(): - return self.hc_stats_stream - # Verify batches can also compute coefficients beside the - # sublayer; each branch joins before hc_post reads those coefficients. + @functools.cached_property + def _can_fuse_ffn_mhc(self) -> bool: + """The MoE counterpart of `_can_fuse_attn_mhc`.""" return ( - self.hc_stats_stream - if ( - forward_batch.forward_mode.is_decode() - or ( - forward_batch.forward_mode.is_target_verify() - and hidden_states.shape[0] > 0 - ) - ) - and ( - not get_platform().is_sm90 - or hidden_states.shape[0] == 1 - or ( - self.config.model_type == "deepseek_v41" - and forward_batch.forward_mode.is_decode() - and 1 < hidden_states.shape[0] <= 64 - ) - ) - else None + self.mlp.tp_size == 4 + and get_moe_a2a_backend().is_none() + and not self.mlp._shared_expert_tp1 + and not should_skip_post_experts_all_reduce(is_tp_path=True) + and mhc.can_fuse_post(self.hc_cfg) ) - def _hc_post_with_combine( - self, x, residual, post, comb, pre, forward_batch, norm=None - ): - """Return updated HC streams and optional combined/normalized inputs.""" - from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled - - if ( - self.config.model_type == "deepseek_v41" - and x.is_cuda - and get_platform().is_blackwell - and ( - ( - 128 <= x.shape[0] <= 384 - and ( - forward_batch.forward_mode.is_decode() - or forward_batch.forward_mode.is_target_verify() - ) - ) - or ( - 4096 <= x.shape[0] <= 65536 - and forward_batch.forward_mode.is_extend_without_speculative() - and envs.SGLANG_OPT_USE_TILELANG_MHC_POST.get() - and not envs.SGLANG_OPT_USE_FLASHINFER_MHC.get() - ) - ) - and x.shape[1] == 5120 - and self.hc_mult == 4 - and x.dtype == residual.dtype == torch.bfloat16 - and post.dtype == comb.dtype == pre.dtype == torch.float32 - and all(t.is_contiguous() for t in (x, residual, post, comb, pre)) - and get_parallel().attn_dp_size == 1 - and not get_forward().sp_active - and not self.dsa_enable_prefill_cp - and not is_batch_invariant_mode_enabled() - ): - if ( - x.shape[0] >= 4096 - and norm is not None - and not norm.cast_x_before_out_mul - and norm.variance_size_override is None - and norm.weight.dtype == torch.bfloat16 - and norm.weight.shape == (5120,) - and norm.weight.is_contiguous() - and all(t.data_ptr() % 16 == 0 for t in (x, residual, norm.weight)) - ): - from sglang.kernels.ops.layernorm.mhc_post_combine_norm_prefill import ( - mhc_post_combine_norm_prefill, - ) - - updated, normalized = mhc_post_combine_norm_prefill( - x, residual, post, comb, pre, norm.weight, norm.variance_epsilon - ) - return updated, None, normalized - from sglang.kernels.ops.layernorm.mhc_post_combine import mhc_post_combine - - updated, combined = mhc_post_combine(x, residual, post, comb, pre) - return updated, combined, None - return self.hc_post(x, residual, post, comb), None, None - def forward_hc_pre_from_prev( self, positions: torch.Tensor, - hidden_states: torch.Tensor, + state: mhc.HcState, input_ids: torch.Tensor, forward_batch: ForwardBatch, input_ids_global: torch.Tensor, - prev_pre: Optional[torch.Tensor], - precomputed_attn: Optional[tuple] = None, - next_norm: Optional[RMSNorm] = None, - next_input: Optional[list] = None, - combined_attn: Optional[torch.Tensor] = None, - normalized_attn: Optional[torch.Tensor] = None, - next_combined: Optional[list] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Layer forward where each sublayer consumes the previous sublayer's - pre-mix. Returns (hidden_states, ffn_pre).""" - from functools import partial + seam_open: bool = True, + ) -> mhc.HcState: + """The layer's two hyper-connections, each collapsing with the previous one's + pre-mix. ``seam_open`` is False when the late-layer tail narrows the rows after + this layer, so nothing precomputed for the next one would still describe it.""" + self._init_boundaries() + stats_stream = None + if mhc.use_stats_stream(self.hc_cfg, forward_batch, state.residual): + stats_stream = self.hc_stats_stream - from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled - from sglang.srt.layers.moe.mhc_post_fusion import ( - MhcPostFusion, - use_mhc_post_fusion, - ) - - stats_stream = self._get_hc_stats_stream(hidden_states, forward_batch) - residual = hidden_states - attn_quantized: Optional[list] = ( - [] if self.self_attn.accepts_mxfp8_swizzled_input() else None - ) - attn_stats = partial( - self._hc_mix_stats, - hidden_states, - self.hc_attn_fn, - self.hc_attn_scale, - self.hc_attn_base, - stats_stream, - ) - x = self._hc_combine( - hidden_states, - apply_pre=prev_pre, - norm=self.input_layernorm, - stats_stream=stats_stream, - quantized=attn_quantized, - precomputed=precomputed_attn, - combined=combined_attn, - normalized=normalized_attn, - ) - prefill_overlap = ( - stats_stream is not None - and forward_batch.forward_mode.is_extend_without_speculative() - ) - medium_verify = 128 <= x.shape[0] <= 384 and ( - forward_batch.forward_mode.is_decode() - or forward_batch.forward_mode.is_target_verify() - ) - attn_mhc = None - if ( - self.config.model_type == "deepseek_v41" - and x.is_cuda - and get_platform().is_blackwell - and (0 < x.shape[0] <= 8 or medium_verify) - and x.shape[1] == 5120 - and self.hc_mult == 4 - and x.dtype == residual.dtype == torch.bfloat16 - and residual.is_contiguous() - and get_parallel().attn_dp_size == 1 - and get_parallel().tp_size == self.self_attn.attn_tp_size == 4 - and self.self_attn.wo_b.reduce_results - and not get_forward().sp_active - and not self.dsa_enable_prefill_cp - and not self.post_attention_layernorm.cast_x_before_out_mul - and self.post_attention_layernorm.variance_size_override is None - and self.post_attention_layernorm.weight.dtype == torch.bfloat16 - ): - from sglang.kernels.ops.communication.all_reduce_fusion import ( - get_registered_comm, + def run_attn_hc(state: mhc.HcState) -> mhc.HcState: + assert self.attn_hc is not None + residual = state.residual + quantized = [] if self.self_attn.accepts_mxfp8_swizzled_input() else None + mhc.fork_stats_stream(stats_stream) + x = mhc.combine(self.attn_hc, state, quantized) + state.release() + del state + world_size = self.self_attn.attn_tp_size + fuse_all_reduce_mhc = self._can_fuse_attn_mhc and can_fuse_all_reduce( + x.shape[0], self.hc_cfg.hidden ) - - comm_ready = get_registered_comm(self.self_attn.attn_tp_size) is not None - if medium_verify and not is_batch_invariant_mode_enabled(): - from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( - _fused_finalize_all_reduce_comm_world_size, + with self.self_attn.maybe_use_decode_attn_tp(forward_batch): + y = self.self_attn( + x=x, + positions=positions, + forward_batch=forward_batch, + x_quant=quantized[0] if quantized else None, + defer_all_reduce=fuse_all_reduce_mhc, ) - - comm_ready = ( - _fused_finalize_all_reduce_comm_world_size() - == self.self_attn.attn_tp_size - ) - if not is_batch_invariant_mode_enabled() and comm_ready: - attn_mhc = MhcPostFusion( - residual, - None, - None, - stats_stream, - record_stats=attn_stats, - norm_weight=self.post_attention_layernorm.weight, - norm_eps=self.post_attention_layernorm.variance_epsilon, - combine_only=medium_verify, - ) - if ( - prefill_overlap - and get_parallel().tp_size == self.self_attn.attn_tp_size == 4 - and self.self_attn.wo_b.reduce_results - ): - attn_mhc = MhcPostFusion( + del x + return mhc.run_attn_post( + self.attn_hc, + y, residual, - None, - None, - stats_stream, - overlap_only=True, - record_stats=attn_stats, + stats_stream=stats_stream, + next=self.local_boundary, + world_size=world_size, ) - context = ( - use_mhc_post_fusion(attn_mhc) if attn_mhc is not None else nullcontext() - ) - with context, self.self_attn.maybe_use_decode_attn_tp(forward_batch): - x = self.self_attn( - x=x, - positions=positions, - forward_batch=forward_batch, - x_quant=attn_quantized[0] if attn_quantized else None, + + def run_ffn_hc(state: mhc.HcState) -> mhc.HcState: + assert self.ffn_hc is not None + residual = state.residual + mhc.fork_stats_stream(stats_stream) + x = mhc.combine(self.ffn_hc, state) + state.release() + del state + nxt = self.next_boundary if seam_open else None + fuse_all_reduce_mhc = self._can_fuse_ffn_mhc and can_fuse_all_reduce( + x.shape[0], self.hc_cfg.hidden ) - ffn_combined = None - ffn_normalized = None - if attn_mhc is not None: - attn_mhc.materialize_stats() - if attn_mhc is not None and attn_mhc.output is not None: - attn_pre = attn_mhc.pre - hidden_states = attn_mhc.output - ffn_combined = attn_mhc.combined - ffn_normalized = attn_mhc.normalized - else: - attn_pre, attn_post, attn_comb = ( - (attn_mhc.pre, attn_mhc.post, attn_mhc.comb) - if attn_mhc is not None - else attn_stats() - ) - if stats_stream is not None: - torch.cuda.current_stream().wait_stream(stats_stream) - hidden_states, ffn_combined, ffn_normalized = self._hc_post_with_combine( + y = self._run_moe_ffn_dp_sync( x, - residual, - attn_post, - attn_comb, - attn_pre, forward_batch, - norm=self.post_attention_layernorm, + input_ids=input_ids, + input_ids_global=input_ids_global, + return_moe_output=fuse_all_reduce_mhc, + ) + del x + return mhc.run_moe_post( + self.ffn_hc, + y, + residual, + stats_stream=stats_stream, + next=nxt, + world_size=self.mlp.tp_size, ) - residual = hidden_states - ffn_stats = partial( - self._hc_mix_stats, - hidden_states, - self.hc_ffn_fn, - self.hc_ffn_scale, - self.hc_ffn_base, - stats_stream, - ) - x = self._hc_combine( - hidden_states, - apply_pre=attn_pre, - norm=self.post_attention_layernorm, - stats_stream=stats_stream, - normalized=ffn_normalized, - combined=ffn_combined, - ) - mhc = None - if ( - self.config.model_type == "deepseek_v41" - and x.is_cuda - and get_platform().is_blackwell - and (0 < x.shape[0] <= 8 or (medium_verify and next_combined is not None)) - and x.shape[1] == 5120 - and self.hc_mult == 4 - and x.dtype == residual.dtype == torch.bfloat16 - and residual.is_contiguous() - and get_parallel().attn_dp_size == 1 - and get_moe_a2a_backend().is_none() - and not get_forward().sp_active - and not self.dsa_enable_prefill_cp - and not self.mlp._shared_expert_tp1 - and self.mlp.tp_size == 4 - and (not medium_verify or not is_batch_invariant_mode_enabled()) - ): - mhc = MhcPostFusion( - residual, - None, - None, - stats_stream, - record_stats=ffn_stats, - combine_only=medium_verify, - ) - if next_norm is not None: - mhc.norm_weight = next_norm.weight - mhc.norm_eps = next_norm.variance_epsilon - if ( - prefill_overlap - and self.mlp.tp_size == 4 - and get_moe_a2a_backend().is_none() - ): - mhc = MhcPostFusion( - residual, - None, - None, - stats_stream, - overlap_only=True, - record_stats=ffn_stats, - ) - context = use_mhc_post_fusion(mhc) if mhc is not None else nullcontext() - with context: - x = self._run_moe_ffn_dp_sync( - x, forward_batch, input_ids=input_ids, input_ids_global=input_ids_global - ) - if mhc is not None: - mhc.materialize_stats() - ffn_pre, ffn_post, ffn_comb = mhc.pre, mhc.post, mhc.comb - else: - ffn_pre, ffn_post, ffn_comb = ffn_stats() - if mhc is not None and mhc.output is not None: - hidden_states = mhc.output - if next_input is not None and mhc.quantized is not None: - next_input.append((mhc.normalized, Mxfp8SwizzledInput(*mhc.quantized))) - if next_combined is not None and mhc.combined is not None: - next_combined.append((mhc.combined, None)) - else: - if stats_stream is not None: - torch.cuda.current_stream().wait_stream(stats_stream) - if next_combined is not None: - hidden_states, combined, normalized = self._hc_post_with_combine( - x, - residual, - ffn_post, - ffn_comb, - ffn_pre, - forward_batch, - norm=next_norm, - ) - if combined is not None or normalized is not None: - next_combined.append((combined, normalized)) - else: - hidden_states = self.hc_post(x, residual, ffn_post, ffn_comb) - return hidden_states, ffn_pre + return run_ffn_hc(run_attn_hc(state)) def _run_moe_ffn_dp_sync( self, @@ -3890,7 +3364,8 @@ class DeepseekV4DecoderLayer(nn.Module): *, input_ids: Optional[torch.Tensor], input_ids_global: Optional[torch.Tensor], - ) -> torch.Tensor: + return_moe_output: bool = False, + ) -> Union[torch.Tensor, deepseek_v2.MoEOutput]: _use_cp = is_cp_active(forward_batch) _use_tp_moe_gather = ( not _use_cp @@ -3999,13 +3474,21 @@ class DeepseekV4DecoderLayer(nn.Module): get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter), gathered_rows, ): + return_moe_output = ( + return_moe_output + and not _use_cp + and not _use_tp_moe_gather + and not _use_tp_attn_a2a_scatter + ) hidden_states = self.mlp( hidden_states, forward_batch, input_ids=input_ids, input_ids_global=input_ids_global, skip_shared_experts=_do_shared_local, + return_moe_output=return_moe_output, ) + if _use_cp and get_moe_a2a_backend().is_none(): hidden_states = attn_cp_interleave_reduce_scatter(hidden_states) elif _use_tp_moe_gather: @@ -4461,7 +3944,7 @@ class DeepseekV4Model(nn.Module): ): if x.numel() > 0: if _is_xpu: - return _get_mhc_ops().fused_hc_head( + return get_mhc_ops().fused_hc_head( x.contiguous(), hc_fn, hc_scale, @@ -4556,25 +4039,17 @@ class DeepseekV4Model(nn.Module): attn_backend = get_attn_backend() tail = attn_backend.tail_forward_metadata.late_layer_tail saved_full = None - prev_pre = None - precomputed_attn = None - combined_attn = None - normalized_attn = None - # HIP: the fused boundary hands the next layer its FFN hc_post unapplied - pending_post = None + # A pending post never meets a residual reader: the HIP boundary's defer_post + # gate excludes Engram/DSpark-capture layers and the model end. + # mHC assumes full token rows per rank; LayerNorm SP needs its own path. + assert not get_forward().sp_active + state = mhc.HcState(hidden_states) for i in range(self.start_layer, self.end_layer): if tail is not None and i == self.late_layer_start: - combined_attn = None - normalized_attn = None # Decode reaches back at most SWA_WINDOW positions. saved_full = attn_backend.enter_late_layer_tail(forward_batch) - if pending_post is not None: - # the fused boundary carries the residual stream here, not in hidden_states - pending_post = tuple(tail.rows(t) for t in pending_post) - else: - hidden_states = tail.rows(hidden_states) - prev_pre, input_ids, input_ids_global = ( - tail.rows(prev_pre), + state = state.take_rows(tail.rows) + input_ids, input_ids_global = ( tail.rows(input_ids), tail.rows(input_ids_global), ) @@ -4583,12 +4058,9 @@ class DeepseekV4Model(nn.Module): hash_ids = tail.rows(hash_ids) engram = self.layers[i].engram if engram is not None: - precomputed_attn = None - combined_attn = None - normalized_attn = None - before_engram = hidden_states + before_engram = state.residual hidden_states = engram( - hidden_states, + state.residual, hash_ids[:, engram.layer_hash_index], forward_batch, cp_all_tokens=cp_extend, @@ -4602,9 +4074,10 @@ class DeepseekV4Model(nn.Module): before_engram, hidden_states, ) + state = state.with_residual(hidden_states) if capture_dspark and i in self.dspark_layers_to_capture: # The draft head reads the attention input of its target layers. - aux = hidden_states + aux = state.residual if tail is not None and i < self.late_layer_start: aux = tail.rows(aux) dspark_aux_hidden_states.append(aux.mean(dim=1)) @@ -4613,102 +4086,32 @@ class DeepseekV4Model(nn.Module): if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE) else get_global_expert_distribution_recorder().with_current_layer(i) ) - next_norm = None - next_input = [] - # The next layer can consume a collapsed input only if no Engram - # or row selection changes the residual between the two layers. - # The HIP fused boundary carries the residual in pending_post and - # leaves hidden_states None; it never consumes a collapsed input. - next_combined = ( - [] - if ( - hidden_states is not None - and self.config.model_type == "deepseek_v41" - and ( - 128 <= hidden_states.shape[0] <= 384 - or ( - 4096 <= hidden_states.shape[0] <= 65536 - and forward_batch.forward_mode.is_extend_without_speculative() - ) - ) - and i + 1 < self.end_layer - and tail is None - and self.layers[i + 1].engram is None - ) - else None - ) - if next_combined is not None and hidden_states.shape[0] >= 4096: - next_norm = self.layers[i + 1].input_layernorm - if ( - self.config.model_type == "deepseek_v41" - and get_platform().is_blackwell - and i + 1 < self.end_layer - and tail is None - and hidden_states.is_cuda - and 0 < hidden_states.shape[0] <= 8 - and ( - forward_batch.forward_mode.is_decode() - or forward_batch.forward_mode.is_target_verify() - ) - and not get_forward().sp_active - and self.layers[i + 1].engram is None - and self.layers[i + 1].self_attn.accepts_mxfp8_swizzled_input() - ): - from sglang.srt.batch_invariant_ops import ( - is_batch_invariant_mode_enabled, - ) - - norm = self.layers[i + 1].input_layernorm - if ( - not norm.cast_x_before_out_mul - and norm.variance_size_override is None - and norm.weight.dtype == torch.bfloat16 - and norm.weight.shape == (5120,) - and norm.weight.is_contiguous() - and not is_batch_invariant_mode_enabled() - ): - next_norm = norm with ctx: if _is_hip and self.layers[i].hc_boundary_fused: - hidden_states, prev_pre, pending_post = ( - _hip.forward_layer_fused_boundary( - self, - i, - positions=positions, - hidden_states=hidden_states, - input_ids=input_ids, - forward_batch=forward_batch, - input_ids_global=input_ids_global, - prev_pre=prev_pre, - pending_post=pending_post, - capture_dspark=capture_dspark, - ) + state = _hip.forward_layer_fused_boundary( + self, + i, + state, + positions=positions, + input_ids=input_ids, + forward_batch=forward_batch, + input_ids_global=input_ids_global, + capture_dspark=capture_dspark, ) - # the fused boundary never publishes a collapsed input for the next layer - precomputed_attn = combined_attn = normalized_attn = None continue - hidden_states, prev_pre = self.layers[i].forward_hc_pre_from_prev( + state = self.layers[i].forward_hc_pre_from_prev( positions=positions, - hidden_states=hidden_states, + state=state, input_ids=input_ids, forward_batch=forward_batch, input_ids_global=input_ids_global, - prev_pre=prev_pre, - precomputed_attn=precomputed_attn, - next_norm=next_norm, - next_input=next_input, - combined_attn=combined_attn, - normalized_attn=normalized_attn, - next_combined=next_combined, + seam_open=tail is None, ) - precomputed_attn = next_input[0] if next_input else None - combined_attn, normalized_attn = ( - next_combined[0] if next_combined else (None, None) - ) + state = state.materialized(self.layers[self.end_layer - 1].hc_cfg) if saved_full is not None: attn_backend.exit_late_layer_tail(saved_full, forward_batch) - return hidden_states, prev_pre, tail - return hidden_states, prev_pre, None + return state.residual, state.pre, tail + return state.residual, state.pre, None def _can_run_tbo(self, forward_batch: ForwardBatch) -> bool: """DSV4 prefill-only two-batch-overlap gate. @@ -5372,6 +4775,10 @@ class DeepseekV4ForCausalLM(nn.Module): ): self_attn.indexer.compressor.apply_ape_hotfix() layer.refresh_mhc_norm_weight_cache() + layers = self.model.layers + for i, layer in enumerate(layers): + if isinstance(layer, DeepseekV4DecoderLayer): + layer.bind_next(layers[i + 1] if i + 1 < len(layers) else None) def precompile_kernels_after_loading(self) -> None: from sglang.srt.layers.moe.mega_moe import ( @@ -5379,6 +4786,15 @@ class DeepseekV4ForCausalLM(nn.Module): should_fuse_mega_moe_shared_experts, ) + # The only deployments whose fused-collective gates can fire; registration + # must happen eagerly here, before graph capture. + if get_platform().is_blackwell and get_parallel().tp_size == 4: + from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( + register_fused_all_reduce_comm, + ) + + register_fused_all_reduce_comm() + for module in self.modules(): if isinstance( module, deepseek_v2.DeepseekV2MoE diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index f2f3ccbc25..1c6c29dc80 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -35,6 +35,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.forward_context import get_token_to_kv_pool from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.models import deepseek_v4_mhc as mhc from sglang.srt.models.dbrx import ReplicatedLinear from sglang.srt.models.deepseek_v4 import ( DEEPSEEK_V4_STACKED_PARAMS_MAPPING, @@ -745,35 +746,31 @@ class DSparkV4Stage(DeepseekV4DecoderLayer): forward_batch: ForwardBatch, prev_pre: Optional[torch.Tensor], ) -> Tuple[torch.Tensor, torch.Tensor]: - stats_stream = self._get_hc_stats_stream(hidden_states, forward_batch) + # The draft stages run the plain four steps: no fused post, no seams, so + # they need nothing from `bind_next` and never hand anything forward. + assert self.attn_hc is not None and self.ffn_hc is not None + stats_stream = None + if mhc.use_stats_stream(self.hc_cfg, forward_batch, hidden_states): + stats_stream = self.hc_stats_stream + residual = hidden_states - x = self._hc_combine( - hidden_states, prev_pre, self.input_layernorm, stats_stream - ) + mhc.fork_stats_stream(stats_stream) + x = mhc.combine(self.attn_hc, mhc.HcState(hidden_states, prev_pre)) with self.self_attn.maybe_use_decode_attn_tp(forward_batch): x = self.self_attn(positions, x, forward_batch) - attn_pre, attn_post, attn_comb = self._hc_mix_stats( - hidden_states, - self.hc_attn_fn, - self.hc_attn_scale, - self.hc_attn_base, - stats_stream, + attn_pre, attn_post, attn_comb = mhc.mix_stats( + self.attn_hc, hidden_states, stats_stream ) if stats_stream is not None: torch.cuda.current_stream().wait_stream(stats_stream) hidden_states = self.hc_post(x, residual, attn_post, attn_comb) residual = hidden_states - x = self._hc_combine( - hidden_states, attn_pre, self.post_attention_layernorm, stats_stream - ) + mhc.fork_stats_stream(stats_stream) + x = mhc.combine(self.ffn_hc, mhc.HcState(hidden_states, attn_pre)) x = self._run_ffn(x, forward_batch) - ffn_pre, ffn_post, ffn_comb = self._hc_mix_stats( - hidden_states, - self.hc_ffn_fn, - self.hc_ffn_scale, - self.hc_ffn_base, - stats_stream, + ffn_pre, ffn_post, ffn_comb = mhc.mix_stats( + self.ffn_hc, hidden_states, stats_stream ) if stats_stream is not None: torch.cuda.current_stream().wait_stream(stats_stream) diff --git a/python/sglang/srt/models/deepseek_v4_mhc.py b/python/sglang/srt/models/deepseek_v4_mhc.py new file mode 100644 index 0000000000..a0a66dc772 --- /dev/null +++ b/python/sglang/srt/models/deepseek_v4_mhc.py @@ -0,0 +1,715 @@ +"""DeepSeek-V4.1 manifold-constrained hyper-connections (mHC). + +The residual is ``R [T, hc_mult, hidden]`` -- hc_mult parallel streams per token -- +and each sublayer is wrapped in a hyper-connection (DeepSeek's ``attn_hc`` and +``ffn_hc``) carrying a ``(pre, post, comb)`` triplet. Per hyper-connection: + + 1. mix stats predict ``(pre, post, comb)`` from ``R`` + 2. combine collapse the streams with the PREVIOUS triplet's ``pre``, then norm + 3. sublayer attention or MoE, including its tensor-parallel reduction + 4. post ``R'[i] = post[i] * y + sum_j comb[j][i] * R[j]`` + +Step 1's ``pre`` is consumed by step 2 of the *next* hyper-connection -- the lag +that lets step 1 run beside steps 2-3 and makes V4's fused post+pre inexpressible. +``attn_hc -> ffn_hc`` stays inside a layer; ``ffn_hc -> attn_hc`` crosses layers, +where an Engram or a row selection can replace ``R`` (`with_residual`/`take_rows`). + +The layer owns parameters, parallelism and token-range policy; this module owns +the kernel dispatch. + +TODO: move dsv4 MHC into the same abstraction +""" + +from __future__ import annotations + +from typing import Any, Callable, NamedTuple, Optional, Tuple, TypeAlias, Union + +import msgspec +import torch +import torch.nn.functional as F + +from sglang.kernels.ops.layernorm.mhc_post_split_h import mhc_post_split_h +from sglang.srt.environ import envs +from sglang.srt.hardware_backend.npu.utils import is_npu_arch35 +from sglang.srt.layers.layernorm import RMSNorm +from sglang.srt.layers.quantization.mxfp8_input import Mxfp8SwizzledInput +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.runner import compile_in_capture_mode +from sglang.srt.models.deepseek_v2 import MoEOutput, _is_hip, _is_npu, _is_xpu +from sglang.srt.runtime_context import get_parallel, get_platform +from sglang.srt.utils import is_gfx95_supported + +_is_gfx95_supported = is_gfx95_supported() + +# --------------------------------------------------------------------------- +# What a layer hands this module +# --------------------------------------------------------------------------- + +HcTriplet = Tuple[torch.Tensor, torch.Tensor, torch.Tensor] + + +class HcConfig(NamedTuple): + """Everything about a layer's mHC that is fixed for the model's lifetime.""" + + mult: int + sinkhorn_iters: int + eps: float + rms_eps: float + hidden: int + pre_from_prev: bool + cp_prefill: bool + + +class HcSubLayer(NamedTuple): + """One hyper-connection's fixed material. Holds the layer's Parameters by + reference; a plain tuple registers nothing, so checkpoint keys stay put.""" + + cfg: HcConfig + fn: torch.Tensor + scale: torch.Tensor + base: torch.Tensor + norm: RMSNorm + tf32_parts: Optional[Any] = None + bf16_parts: Optional[Any] = None + + +# --------------------------------------------------------------------------- +# How far the previous hyper-connection got toward the next sublayer's input +# --------------------------------------------------------------------------- + + +class HcNormed(NamedTuple): + """Summed and normed; the sublayer can take these rows as they are.""" + + rows: torch.Tensor + + +class HcQuantized(NamedTuple): + """Normed, and already swizzled for the sublayer's first projection.""" + + rows: torch.Tensor + swizzled: Mxfp8SwizzledInput + + +HcPreOutput: TypeAlias = Union[HcNormed, HcQuantized] +"""The next sublayer's input, built by the previous post; a receiver skips step 2 +(and ``pre``) entirely. Always normed -- combine and norm travel together.""" + + +class AttnOutput(NamedTuple): + """Attention's wo_b rows with the TP reduction left to the fused post.""" + + partial: torch.Tensor + + +# --------------------------------------------------------------------------- +# The real parameters: what crosses from one hyper-connection to the next +# --------------------------------------------------------------------------- + + +class HcPending(NamedTuple): + """A post not run yet; the residual it would produce is implied. The backward + counterpart of `HcPreOutput`: the HIP boundary kernel takes both and runs the + previous post, this triplet and this collapse in one launch.""" + + rows: torch.Tensor + residual: torch.Tensor + post: torch.Tensor + comb: torch.Tensor + + +class HcState(msgspec.Struct): + """The mHC state of the stream between two hyper-connections. + + ``streams`` is the residual ``R``, or -- out of the HIP fused boundary -- the + post that would rebuild it, still outstanding. Mutable for one reason: + `release` must empty a consumed state in every frame that still binds it. + """ + + streams: Union[torch.Tensor, HcPending, None] + pre: Optional[torch.Tensor] = None + input: Optional[HcPreOutput] = None + + def release(self) -> None: + """Empty the state at its last reader, the combine; the residual the post + still needs lives on through the caller's alias. Stale holders -- above + all the model loop's binding across the layer call -- then stop pinning + the dead residual through the next sublayer.""" + self.streams = None + self.pre = None + self.input = None + + @property + def residual(self) -> torch.Tensor: + """The materialized streams; run an outstanding post first (`materialized`).""" + assert isinstance(self.streams, torch.Tensor) + return self.streams + + def materialized(self, cfg: HcConfig) -> HcState: + """Run the outstanding post, if any; residual readers call this first.""" + if not isinstance(self.streams, HcPending): + return self + p = self.streams + return HcState(post(cfg, p.rows, p.residual, p.post, p.comb), self.pre) + + def with_residual(self, residual: torch.Tensor) -> HcState: + """The residual was rewritten: drop ``input``; ``pre`` weights streams, not + rows, so it survives.""" + return HcState(residual, self.pre) + + def take_rows(self, rows: Callable[[torch.Tensor], torch.Tensor]) -> HcState: + """Keep only the rows ``rows`` selects -- the late-layer tail's narrowing.""" + pre = None if self.pre is None else rows(self.pre) + assert self.streams is not None + if isinstance(self.streams, HcPending): + return HcState(HcPending(*(rows(t) for t in self.streams)), pre) + else: + return HcState(rows(self.streams), pre) + + +# --------------------------------------------------------------------------- +# The plan: how a hyper-connection intends to run its post +# --------------------------------------------------------------------------- + + +class HcNextBoundary(NamedTuple): + """What the next hyper-connection can take off this one's post; resolved after + weights load. None where the seam is closed (last layer, Engram).""" + + norm: RMSNorm + accepts_mxfp8: bool + norm_fusable: bool + + +def is_deferred_finalize(routed) -> bool: + """Whether a `MoEOutput` really carries an unfinalized handle for the post.""" + from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( + FlashInferTrtllmDeferredFinalizeOutput, + ) + + return isinstance(routed, FlashInferTrtllmDeferredFinalizeOutput) + + +def make_boundary(norm: RMSNorm, *, accepts_mxfp8: bool) -> HcNextBoundary: + """Can this hyper-connection's norm fold into the previous post, and does its + sublayer take a pre-quantized input.""" + return HcNextBoundary( + norm=norm, + accepts_mxfp8=accepts_mxfp8, + norm_fusable=( + not norm.cast_x_before_out_mul + and norm.variance_size_override is None + and norm.weight.dtype == torch.bfloat16 + # TODO: support other norm shape + and norm.weight.shape == (5120,) + and norm.weight.is_contiguous() + ), + ) + + +# --------------------------------------------------------------------------- +# Step 1 -- the mix GEMM and the triplet it splits into +# --------------------------------------------------------------------------- + + +def use_stats_stream(cfg: HcConfig, forward_batch: ForwardBatch, x: torch.Tensor): + if not cfg.pre_from_prev: + return False + decode_like = forward_batch.forward_mode.is_decode() or ( + forward_batch.forward_mode.is_target_verify() and x.shape[0] > 0 + ) + return decode_like and ( + not get_platform().is_sm90 + or x.shape[0] == 1 + or (forward_batch.forward_mode.is_decode() and 1 < x.shape[0] <= 64) + ) + + +def mix_stats( + hc: HcSubLayer, x: torch.Tensor, stats_stream: Optional[torch.cuda.Stream] = None +) -> HcTriplet: + """Predict the triplet on the caller's stream or the stats stream.""" + if stats_stream is None: + return _mix_stats_impl(hc, x) + main_stream = torch.cuda.current_stream() + x.record_stream(stats_stream) + with torch.cuda.stream(stats_stream): + coefficients = _mix_stats_impl(hc, x) + for coefficient in coefficients: + coefficient.record_stream(main_stream) + return coefficients + + +def _mix_stats_impl(hc: HcSubLayer, x: torch.Tensor) -> HcTriplet: + from sglang.kernels.ops.layernorm.mhc import ( + hc_mix_stats, + hc_mix_stats_sinkhorn, + hc_split_sinkhorn, + ) + + cfg = hc.cfg + + x_flat = x.flatten(1) + + from sglang.srt.batch_invariant_ops import ( + is_batch_invariant_mode_enabled, + ) + + parts = bf16_parts = None + hopper_medium = get_platform().is_sm90 and 32 <= x_flat.shape[0] < 4096 + if ( + x.is_cuda + and (x_flat.shape[0] >= 128 or hopper_medium) + and x_flat.is_contiguous() + and (get_platform().is_sm100 or get_platform().is_sm90) + and envs.SGLANG_OPT_DEEPGEMM_HC_PRENORM.get() + and not is_batch_invariant_mode_enabled() + ): + parts, bf16_parts = hc.tf32_parts, hc.bf16_parts + + use_bf16_projection = bf16_parts is not None and ( + hopper_medium or 4096 <= x_flat.shape[0] <= 65536 + ) + hopper_fused_stats = get_platform().is_sm90 and ( + x.shape[0] == 1 or (bf16_parts is not None and 32 <= x.shape[0] <= 65536) + ) + # gfx950 uses the fused Triton port at every row count (MI350X). + use_fused_stats = _is_gfx95_supported or ( + torch.version.cuda is not None + and (get_platform().is_blackwell or hopper_fused_stats) + ) + if x.is_cuda and use_fused_stats and x.dtype == torch.bfloat16: + # The default split-K/Sinkhorn fusion preserves batch invariance; + # compensated projections above are disabled in batch-invariant mode. + if use_bf16_projection: + from sglang.kernels.ops.layernorm.mhc import ( + hc_mix_stats_sinkhorn_bf16x3, + ) + + pre, post, comb = hc_mix_stats_sinkhorn_bf16x3( + x_flat, + bf16_parts, + hc.scale, + hc.base, + cfg.sinkhorn_iters, + cfg.rms_eps, + cfg.eps, + ) + elif parts is not None: + from sglang.kernels.ops.layernorm.mhc import ( + hc_mix_stats_sinkhorn_deepgemm, + ) + + pre, post, comb = hc_mix_stats_sinkhorn_deepgemm( + x_flat, + parts, + hc.scale, + hc.base, + cfg.sinkhorn_iters, + cfg.rms_eps, + cfg.eps, + ) + else: + pre, post, comb = hc_mix_stats_sinkhorn( + x_flat, + hc.fn, + hc.scale, + hc.base, + cfg.mult, + cfg.sinkhorn_iters, + cfg.rms_eps, + cfg.eps, + ) + return pre, post, comb + if x.is_cuda and torch.version.cuda is not None: + # cuBLAS/torch reductions can change order with num_tokens; this kernel + # keeps the mixing and RMS reductions batch-invariant. + mixes = hc_mix_stats(x_flat, hc.fn, cfg.rms_eps).unsqueeze(1) + else: + x_flat = x_flat.float() + rsqrt = torch.rsqrt(x_flat.square().mean(-1, keepdim=True) + cfg.rms_eps) + mixes = (F.linear(x_flat, hc.fn) * rsqrt).unsqueeze(1) + if _is_xpu: + from sglang.srt.models.deepseek_v4 import get_mhc_ops + + split_sinkhorn = get_mhc_ops().hc_split_sinkhorn + else: + split_sinkhorn = hc_split_sinkhorn + pre, post, comb = split_sinkhorn( + mixes, + hc.scale, + hc.base, + cfg.mult, + cfg.sinkhorn_iters, + cfg.eps, + ) + return pre.squeeze(1), post.squeeze(1), comb.squeeze(1) + + +# --------------------------------------------------------------------------- +# Step 2 -- collapse the streams and normalize +# --------------------------------------------------------------------------- + + +def fork_stats_stream(stats_stream: Optional[torch.cuda.Stream]) -> None: + """Let the side stream pick up from here; the layers call it right before + each combine (fork position measured indistinguishable across batch sizes).""" + if stats_stream is not None: + stats_stream.wait_stream(torch.cuda.current_stream()) + + +def combine( + hc: HcSubLayer, + state: HcState, + quantized: Optional[list] = None, +) -> torch.Tensor: + """The sublayer's input: ``norm(sum_k pre[k] * R[k])``, short-circuited by + whatever the previous post left in ``state.input``.""" + shortcut = state.input + if isinstance(shortcut, HcQuantized): + assert quantized is not None + quantized.append(shortcut.swizzled) + return shortcut.rows + if isinstance(shortcut, HcNormed): + return shortcut.rows + return _combine(hc, state, quantized) + + +def _combine( + hc: HcSubLayer, + state: HcState, + quantized: Optional[list], +) -> torch.Tensor: + """The combine computed from the residual's rows.""" + from sglang.kernels.ops.layernorm.mhc import hc_combine + + cfg, norm = hc.cfg, hc.norm + x, apply_pre = state.residual, state.pre + quantize = quantized is not None + x_flat = x.flatten(1) + + if apply_pre is None: + return norm(x[:, 0, :]) + from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled + + hopper_fused = ( + get_platform().is_sm90 + and not quantize + and (0 < x.shape[0] <= 96 or 4096 <= x.shape[0] <= 65536) + and not is_batch_invariant_mode_enabled() + ) + if ( + x.is_cuda + and (get_platform().is_blackwell or hopper_fused) + and x.dtype == torch.bfloat16 + and apply_pre.stride(1) == 1 + and cfg.mult == 4 + and cfg.hidden == 5120 + and norm.weight.dtype == torch.bfloat16 + and not norm.cast_x_before_out_mul + and norm.variance_size_override is None + ): + # One fused form for every row count. Each row's reduction order is + # fixed regardless of the grid split, so this holds under + # batch-invariant mode too. + if quantize and x.shape[0] <= 128: + from sglang.kernels.ops.layernorm.hc_combine_norm import ( + hc_combine_norm_mxfp8, + ) + + y, y_q, y_sf = hc_combine_norm_mxfp8( + x_flat, apply_pre, norm.weight, norm.variance_epsilon + ) + quantized.append(Mxfp8SwizzledInput(y_q, y_sf)) + return y + from sglang.kernels.ops.layernorm.hc_combine_norm import hc_combine_norm + + return hc_combine_norm(x_flat, apply_pre, norm.weight, norm.variance_epsilon) + return norm(hc_combine(x_flat, apply_pre, cfg.mult, x.dtype)) + + +# --------------------------------------------------------------------------- +# Step 4 -- rebuild the streams, optionally collapsing the next input too +# --------------------------------------------------------------------------- + + +def post( + cfg: HcConfig, + x: torch.Tensor, + residual: torch.Tensor, + post_mix: torch.Tensor, + comb: torch.Tensor, +) -> torch.Tensor: + """``R'[i] = post[i] * x + sum_j comb[j][i] * R[j]``, by whichever kernel fits.""" + if x.shape[0] == 0: + return torch.empty((0, cfg.mult, x.shape[-1]), dtype=x.dtype, device=x.device) + + if _is_npu: + if not is_npu_arch35(): + return torch.ops.custom.npu_hc_post(x, residual, post_mix, comb) + # The A5 build of npu_hc_post is batched -- it requires a leading batch axis + # on every operand. + return torch.ops.custom.npu_hc_post( + x.unsqueeze(0), + residual.unsqueeze(0), + post_mix.unsqueeze(0), + comb.unsqueeze(0), + ).squeeze(0) + + if _is_xpu: + from sglang.srt.models.deepseek_v4 import get_mhc_ops + + return get_mhc_ops().mhc_post(x, residual, post_mix, comb) + + if ( + get_platform().is_blackwell + and cfg.pre_from_prev + and cfg.mult == 4 + and cfg.hidden == 5120 + # This 384 is the kernel's own tile limit, unrelated to the collective's + # push slots; it covers every narrow batch, not one particular regime. + and x.shape[0] <= 384 + ): + return mhc_post_split_h(x, residual, post_mix, comb) + + if _is_hip: + from sglang.srt.models.deepseek_common.amd import deepseek_v4_hip as _hip + + y = _hip.hc_post(cfg, x, residual, post_mix, comb) + if y is not None: + return y + + if envs.SGLANG_OPT_USE_FLASHINFER_MHC.get(): + from flashinfer.mhc import mhc_post + + return mhc_post(x, residual, post_mix, comb) + + if envs.SGLANG_OPT_USE_TILELANG_MHC_POST.get(): + if ( + cfg.pre_from_prev + and get_platform().is_sm90 + and cfg.mult == 4 + and cfg.hidden == 5120 + and 1 <= x.shape[0] <= 64 + ): + return mhc_post_split_h(x, residual, post_mix, comb) + + from sglang.kernels.ops.layernorm.mhc import mhc_post + + return mhc_post(x, residual, post_mix, comb) + + if _is_hip: + from aiter.ops.mhc import mhc_post + + result = torch.empty_like(residual) + mhc_post(result, x, residual, post_mix, comb) + return result + + assert residual.shape == (x.shape[0], cfg.mult, x.shape[-1]) + assert post_mix.shape == (x.shape[0], cfg.mult) + assert comb.shape == (x.shape[0], cfg.mult, cfg.mult) + + @compile_in_capture_mode + def post_torch_impl(x, residual, post_mix, comb): + return ( + post_mix.unsqueeze(-1) * x.unsqueeze(1) + + (comb.unsqueeze(-1) * residual.unsqueeze(2)).sum(dim=1) + ).type_as(x) + + return post_torch_impl(x, residual, post_mix, comb) + + +# --------------------------------------------------------------------------- +# Planning the post +# --------------------------------------------------------------------------- + + +def can_fuse_post(cfg: HcConfig) -> bool: + """Whether a fused post is expressible at all for this model, device and + parallel layout. + + The capability half of the decision; which token ranges should take it, and what + batch-invariant mode should do about it, are the caller's policy. + """ + return ( + get_platform().is_blackwell + and cfg.pre_from_prev + and cfg.mult == 4 + and cfg.hidden == 5120 + and get_parallel().attn_dp_size == 1 + and not cfg.cp_prefill + ) + + +# --------------------------------------------------------------------------- +# Running the post +# --------------------------------------------------------------------------- + + +def _compute_triplet( + hc: HcSubLayer, + residual: torch.Tensor, + stats_stream: Optional[torch.cuda.Stream], +) -> HcTriplet: + """Issue the triplet (on the side stream, beside the sublayer) and join before + the post reads it.""" + coefficients = mix_stats(hc, residual, stats_stream) + if stats_stream is not None: + torch.cuda.current_stream().wait_stream(stats_stream) + return coefficients + + +def _post_fusion( + hc: HcSubLayer, + y: torch.Tensor, + residual: torch.Tensor, + coefficients: HcTriplet, + next: Optional[HcNextBoundary], +) -> HcState: + """Step 4 without a collective: the wide-tile kernel folds the next combine + + norm in where it serves this seam, otherwise the pure post runs and the next + combine computes itself.""" + cfg = hc.cfg + pre, post_mix, comb = coefficients + if ( + next is not None + and next.norm_fusable + and cfg.pre_from_prev + and get_platform().is_blackwell + and 4096 <= y.shape[0] <= 65536 + and cfg.hidden == 5120 + and cfg.mult == 4 + and get_parallel().attn_dp_size == 1 + and not cfg.cp_prefill + ): + from sglang.kernels.ops.layernorm.mhc_post_combine_norm_prefill import ( + mhc_post_combine_norm_prefill, + ) + + updated, normalized = mhc_post_combine_norm_prefill( + y, + residual, + post_mix, + comb, + pre, + next.norm.weight, + next.norm.variance_epsilon, + ) + return HcState(updated, pre, HcNormed(normalized)) + return HcState(post(cfg, y, residual, post_mix, comb), pre) + + +def run_attn_post( + hc: HcSubLayer, + out: Union[torch.Tensor, AttnOutput], + residual: torch.Tensor, + *, + stats_stream: Optional[torch.cuda.Stream], + next: Optional[HcNextBoundary], + world_size: int, +) -> HcState: + """The attention post. An `AttnOutput` rides the collective kernel (which also + folds ``next``'s norm); the attention may decline the handover even when asked, + so the type is the ground truth.""" + if isinstance(out, AttnOutput): + from sglang.kernels.ops.communication.all_reduce_mhc import ( + all_reduce_mhc_post_combine_norm, + ) + + assert next is not None + pre, post_mix, comb = _compute_triplet(hc, residual, stats_stream) + _, updated, normalized = all_reduce_mhc_post_combine_norm( + out.partial, + residual, + post_mix, + comb, + pre, + next.norm.weight, + next.norm.variance_epsilon, + world_size=world_size, + ) + return HcState(updated, pre, HcNormed(normalized)) + coefficients = _compute_triplet(hc, residual, stats_stream) + return _post_fusion(hc, out, residual, coefficients, next) + + +def run_moe_post( + hc: HcSubLayer, + out: Union[torch.Tensor, MoEOutput], + residual: torch.Tensor, + *, + stats_stream: Optional[torch.cuda.Stream], + next: Optional[HcNextBoundary], + world_size: int, +) -> HcState: + """The MoE post. A deferred finalize the push plane can carry rides the + collective kernel (finalize + shared add + all-reduce, quantizing ``next``'s + input when a boundary is given); anything else is finalized here and takes + the plain post.""" + from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( + can_fuse_all_reduce, + ) + + if ( + isinstance(out, MoEOutput) + and is_deferred_finalize(out.routed) + # expert_weights rows are the true token count; gemm2_out is the expanded + # [T x top_k (padded)] view and overstates the plane load by ~6x. + and can_fuse_all_reduce(out.routed.expert_weights.shape[0], hc.cfg.hidden) + ): + pre, post_mix, comb = _compute_triplet(hc, residual, stats_stream) + args = ( + out.routed.gemm2_out, + out.routed.expanded_idx_to_permuted_idx, + out.routed.expert_weights, + out.routed.top_k, + out.shared, + residual, + post_mix, + comb, + ) + if ( + next is not None + and next.accepts_mxfp8 + and next.norm_fusable + # The quant epilogue supports at most 128 batch size + and out.routed.expert_weights.shape[0] <= 128 + ): + from sglang.kernels.ops.communication.all_reduce_mhc import ( + moe_finalize_all_reduce_mhc_post_combine_norm_quant, + ) + + _, updated, normalized, q, sf = ( + moe_finalize_all_reduce_mhc_post_combine_norm_quant( + *args, + pre, + next.norm.weight, + next.norm.variance_epsilon, + world_size=world_size, + ) + ) + return HcState( + updated, pre, HcQuantized(normalized, Mxfp8SwizzledInput(q, sf)) + ) + from sglang.kernels.ops.communication.all_reduce_mhc import ( + moe_finalize_all_reduce_mhc_post, + ) + + _, updated = moe_finalize_all_reduce_mhc_post(*args, world_size=world_size) + return HcState(updated, pre) + if isinstance(out, MoEOutput): + from sglang.srt.layers.moe import post_experts_all_reduce + from sglang.srt.layers.moe.utils import should_add_replicated_moe_output + + # The post boundary owns the sum; a replicated shared expert is added + # after the reduction so it is counted only once across TP ranks. + pieces = out + out = post_experts_all_reduce(pieces.get_merged()) + if ( + pieces.shared is not None + and pieces.shared_is_replicated + and should_add_replicated_moe_output() + ): + out += pieces.shared + coefficients = _compute_triplet(hc, residual, stats_stream) + return _post_fusion(hc, out, residual, coefficients, next) diff --git a/test/registered/kernels/ops/layernorm/test_mhc_aiter_hip.py b/test/registered/kernels/ops/layernorm/test_mhc_aiter_hip.py index f68f4f036d..c1ca8321b2 100644 --- a/test/registered/kernels/ops/layernorm/test_mhc_aiter_hip.py +++ b/test/registered/kernels/ops/layernorm/test_mhc_aiter_hip.py @@ -216,9 +216,12 @@ class TestMhcPostSplitH(CustomTestCase): def setUp(self): from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer + from sglang.srt.models.deepseek_v4_mhc import HcConfig self.layer = SimpleNamespace( - config=SimpleNamespace(model_type="deepseek_v41"), hc_mult=4 + config=SimpleNamespace(model_type="deepseek_v41"), + hc_mult=4, + hc_cfg=HcConfig(4, 20, 1e-6, 1e-6, 5120, True, False), ) self.run_post = lambda *args: DeepseekV4DecoderLayer.hc_post(self.layer, *args) for setting in ( diff --git a/test/registered/kernels/ops/layernorm/test_mhc_kernels.py b/test/registered/kernels/ops/layernorm/test_mhc_kernels.py index e100d9e1da..6084fdd4c6 100644 --- a/test/registered/kernels/ops/layernorm/test_mhc_kernels.py +++ b/test/registered/kernels/ops/layernorm/test_mhc_kernels.py @@ -209,24 +209,37 @@ def _check_glm_boundary(x, residual, post, comb, fn, scale, base, *, use_norm): ) -def _hopper_mhc_layer(norm, w): - return SimpleNamespace( +def _hopper_mhc_layer(norm, w, scale, base): + from sglang.srt.models import deepseek_v4_mhc as model_mhc + from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer + + layer = SimpleNamespace( input_layernorm=norm, post_attention_layernorm=norm, config=SimpleNamespace(model_type="deepseek_v41"), hc_pre_from_prev_sublayer=True, hc_attn_fn=w, hc_ffn_fn=-w, + hc_attn_scale=scale, + hc_ffn_scale=scale, + hc_attn_base=base, + hc_ffn_base=base, + hc_cfg=model_mhc.HcConfig(4, 20, 1e-6, 1e-6, 5120, True, False), hc_mult=4, hc_sinkhorn_iters=20, rms_norm_eps=1e-6, hc_eps=1e-6, ) + layer._init_hyper_connections = lambda: ( + DeepseekV4DecoderLayer._init_hyper_connections(layer) + ) + return layer @pytest.mark.parametrize("num_tokens", [32, 33, 64, 128, 257, 4096, 4097, 8192]) def test_hopper_compensated_mhc(num_tokens): from sglang.srt.environ import envs + from sglang.srt.models import deepseek_v4_mhc as model_mhc from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer from sglang.srt.utils import is_sm90_supported @@ -238,12 +251,12 @@ def test_hopper_compensated_mhc(num_tokens): scale = torch.tensor([0.5, 0.25, 0.25], device="cuda") base = torch.randn(24, device="cuda") norm = SimpleNamespace(weight=torch.ones(5120, device="cuda", dtype=torch.bfloat16)) - layer = _hopper_mhc_layer(norm, w) + layer = _hopper_mhc_layer(norm, w, scale, base) with envs.SGLANG_OPT_DEEPGEMM_HC_PRENORM.override(True): DeepseekV4DecoderLayer.refresh_mhc_norm_weight_cache(layer) assert layer._hc_attn_bf16_parts is not None assert layer._hc_attn_tf32_parts is None - actual = DeepseekV4DecoderLayer._hc_mix_stats(layer, x, w, scale, base) + actual = model_mhc.mix_stats(layer.attn_hc, x) xd = x.flatten(1).double() z = (xd @ w.double().T) * torch.rsqrt(xd.square().mean(-1, keepdim=True) + 1e-6) expected_pre = torch.sigmoid(z[:, :4] * scale[0] + base[:4]) + 1e-6 @@ -258,7 +271,7 @@ def test_hopper_compensated_mhc(num_tokens): for result, expected in zip(actual, (expected_pre, expected_post, comb)): torch.testing.assert_close(result.double(), expected, rtol=1e-5, atol=2e-6) with envs.SGLANG_OPT_DEEPGEMM_HC_PRENORM.override(False): - fallback = DeepseekV4DecoderLayer._hc_mix_stats(layer, x, w, scale, base) + fallback = model_mhc.mix_stats(layer.attn_hc, x) for result, expected in zip(fallback, (expected_pre, expected_post, comb)): torch.testing.assert_close(result.double(), expected, rtol=1e-5, atol=2e-6) DeepseekV4DecoderLayer.refresh_mhc_norm_weight_cache(layer) @@ -268,7 +281,7 @@ def test_hopper_compensated_mhc(num_tokens): @pytest.mark.parametrize("num_tokens", [1, 4, 64, 4096]) def test_hopper_combine_norm(num_tokens): from sglang.srt.layers.layernorm import RMSNorm - from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer + from sglang.srt.models import deepseek_v4_mhc as model_mhc from sglang.srt.utils import is_sm90_supported if not is_sm90_supported(): @@ -277,10 +290,14 @@ def test_hopper_combine_norm(num_tokens): x = torch.randn(num_tokens, 4, 5120, device="cuda", dtype=torch.bfloat16) pre = torch.rand(num_tokens, 4, device="cuda") norm = RMSNorm(5120, eps=1e-6).cuda().bfloat16() - layer = SimpleNamespace( - hc_mult=4, config=SimpleNamespace(model_type="deepseek_v41") + hc = model_mhc.HcSubLayer( + model_mhc.HcConfig(4, 20, 1e-6, 1e-6, 5120, True, False), + None, + None, + None, + norm, ) - actual = DeepseekV4DecoderLayer._hc_combine(layer, x, pre, norm) + actual = model_mhc.combine(hc, model_mhc.HcState(x, pre)) expected = norm(mhc.hc_combine(x.flatten(1), pre, 4, x.dtype)) torch.testing.assert_close(actual, expected, rtol=1 / 128, atol=1e-5) @@ -289,6 +306,7 @@ def test_hopper_combine_norm(num_tokens): def test_hopper_mhc_stats_stream_graph(num_tokens): from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.model_executor.forward_batch_info import ForwardMode + from sglang.srt.models import deepseek_v4_mhc as model_mhc from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer from sglang.srt.utils import is_sm90_supported @@ -301,19 +319,20 @@ def test_hopper_mhc_stats_stream_graph(num_tokens): base = torch.randn(24, device="cuda") pre = torch.rand(num_tokens, 4, device="cuda") norm = RMSNorm(5120, eps=1e-6).cuda().bfloat16() - layer = _hopper_mhc_layer(norm, w) + layer = _hopper_mhc_layer(norm, w, scale, base) layer.hc_stats_stream = torch.cuda.Stream() DeepseekV4DecoderLayer.refresh_mhc_norm_weight_cache(layer) - stream = DeepseekV4DecoderLayer._get_hc_stats_stream( - layer, x, SimpleNamespace(forward_mode=ForwardMode.DECODE) + assert model_mhc.use_stats_stream( + layer.hc_cfg, SimpleNamespace(forward_mode=ForwardMode.DECODE), x ) - assert stream is layer.hc_stats_stream + stream = layer.hc_stats_stream def run(stats_stream): - combined = DeepseekV4DecoderLayer._hc_combine(layer, x, pre, norm, stats_stream) - coefficients = DeepseekV4DecoderLayer._hc_mix_stats( - layer, x, w, scale, base, stats_stream - ) + # The layer forks the side stream right before each combine; the combine + # itself no longer takes a stream. + model_mhc.fork_stats_stream(stats_stream) + combined = model_mhc.combine(layer.attn_hc, model_mhc.HcState(x, pre)) + coefficients = model_mhc.mix_stats(layer.attn_hc, x, stats_stream) if stats_stream is not None: torch.cuda.current_stream().wait_stream(stats_stream) # Read on the consumer stream after the join, as hc_post does. @@ -334,6 +353,81 @@ def test_hopper_mhc_stats_stream_graph(num_tokens): torch.testing.assert_close(result, reference, rtol=0, atol=0) +def test_mhc_refresh_without_deepgemm_prenorm(): + from unittest.mock import patch + + from sglang.srt.environ import envs + from sglang.srt.layers.deep_gemm_wrapper import configurer + from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer + from sglang.srt.runtime_context import override_platform + + norm = SimpleNamespace(weight=torch.ones(5120, device="cuda", dtype=torch.bfloat16)) + w = torch.randn(24, 20480, device="cuda") + scale = torch.ones(3, device="cuda") + base = torch.zeros(24, device="cuda") + layer = _hopper_mhc_layer(norm, w, scale, base) + layer._can_fuse_attn_mhc = layer._can_fuse_ffn_mhc = True + layer.local_boundary = layer.next_boundary = object() + with ( + override_platform(is_sm100=True, is_sm90=False), + envs.SGLANG_OPT_DEEPGEMM_HC_PRENORM.override(True), + patch.object(configurer, "ENABLE_JIT_DEEPGEMM", True), + patch.dict("sys.modules", {"deep_gemm": SimpleNamespace()}), + ): + DeepseekV4DecoderLayer.refresh_mhc_norm_weight_cache(layer) + assert layer.attn_hc.fn is w + assert layer.ffn_hc.fn is layer.hc_ffn_fn + assert layer.attn_hc.tf32_parts is None + assert layer.attn_hc.bf16_parts is None + assert "_can_fuse_attn_mhc" not in layer.__dict__ + assert "_can_fuse_ffn_mhc" not in layer.__dict__ + assert layer.local_boundary is None and layer.next_boundary is None + + +@pytest.mark.parametrize( + "replicated,add_shared,has_shared,expected", + [ + (False, False, True, 16.0), + (True, False, True, 2.0), + (True, True, True, 9.0), + (True, True, False, 2.0), + ], +) +def test_moe_post_reduces_before_adding_replicated_shared( + monkeypatch, replicated, add_shared, has_shared, expected +): + from sglang.srt.layers import moe + from sglang.srt.layers.moe import utils as moe_utils + from sglang.srt.models import deepseek_v4_mhc as model_mhc + + routed = torch.ones(2, 3) + shared = torch.full((2, 3), 7.0) if has_shared else None + pieces = model_mhc.MoEOutput(routed, shared, None, 1.0, replicated) + partial = routed if replicated or shared is None else routed + shared + monkeypatch.setattr(model_mhc.MoEOutput, "get_merged", lambda self: partial) + monkeypatch.setattr(model_mhc, "is_deferred_finalize", lambda value: False) + monkeypatch.setattr( + moe_utils, "should_add_replicated_moe_output", lambda: add_shared + ) + reduced_inputs = [] + + def reduce(value): + reduced_inputs.append(value.clone()) + return value * 2 + + monkeypatch.setattr(moe, "post_experts_all_reduce", reduce) + monkeypatch.setattr(model_mhc, "_compute_triplet", lambda *args: None) + monkeypatch.setattr( + model_mhc, "_post_fusion", lambda hc, value, residual, triplet, *args: value + ) + actual = model_mhc.run_moe_post( + None, pieces, None, stats_stream=None, next=None, world_size=2 + ) + assert len(reduced_inputs) == 1 + torch.testing.assert_close(reduced_inputs[0], partial) + torch.testing.assert_close(actual, torch.full((2, 3), expected)) + + if __name__ == "__main__": import sys diff --git a/test/registered/kernels/ops/layernorm/test_mxfp8_epilogue.py b/test/registered/kernels/ops/layernorm/test_mxfp8_epilogue.py index 0cc156719a..0021dc0a25 100644 --- a/test/registered/kernels/ops/layernorm/test_mxfp8_epilogue.py +++ b/test/registered/kernels/ops/layernorm/test_mxfp8_epilogue.py @@ -25,7 +25,9 @@ HIDDEN = 5120 STREAMS = 4 -@pytest.mark.parametrize("m", [1, 2, 5, 6, 8]) +# 9-128 pins the single-tile scale-swizzle bound hc_combine_norm_mxfp8 was +# widened to; 128 is the last row the swizzle addresses without a block offset. +@pytest.mark.parametrize("m", [1, 2, 5, 6, 8, 9, 17, 33, 96, 127, 128]) @pytest.mark.parametrize("scale", [1e-3, 1.0, 1e3]) @pytest.mark.parametrize("backend", ["cuda", "cute-dsl"]) def test_bitwise_identical_to_norm_then_quantize(m: int, scale: float, backend: str): diff --git a/test/registered/unit/models/test_deepseek_v4_amd_tp4.py b/test/registered/unit/models/test_deepseek_v4_amd_tp4.py index 46ec3658b7..f035cf4791 100644 --- a/test/registered/unit/models/test_deepseek_v4_amd_tp4.py +++ b/test/registered/unit/models/test_deepseek_v4_amd_tp4.py @@ -54,6 +54,7 @@ def group(): class _Projection: reduce_results = True + use_decode_attn_tp = False def __init__(self, group): self.group = group @@ -74,14 +75,15 @@ class _Attention: def maybe_use_decode_attn_tp(self, forward_batch): return nullcontext() - def __call__(self, *, x, **kwargs): + def __call__(self, *, x, defer_all_reduce=False, **kwargs): from sglang.srt.models.deepseek_v4 import MQALayer - return MQALayer._project_wo_b(self, x * (self.rank + 1)) + return MQALayer._project_wo_b(self, x * (self.rank + 1), defer_all_reduce) def _layer(group): from sglang.kernels.ops.layernorm.mhc_boundary_hip import rmsnorm_with_sinkhorn + from sglang.srt.models import deepseek_v4_mhc as mhc torch.manual_seed(39186) norm_weight = torch.ones(5120, device="cuda", dtype=torch.bfloat16) @@ -89,6 +91,15 @@ def _layer(group): config=SimpleNamespace(model_type="deepseek_v41"), dsa_enable_prefill_cp=False, hc_pre_from_prev_sublayer=False, + hc_cfg=mhc.HcConfig( + mult=4, + sinkhorn_iters=20, + eps=1e-6, + rms_eps=1e-6, + hidden=5120, + pre_from_prev=True, + cp_prefill=False, + ), self_attn=_Attention(group), mlp=SimpleNamespace(tp_size=1), hc_mult=4, @@ -215,13 +226,13 @@ def _moe_stub(rank, *, dual, shared_tp1): torch.nn.Module.__init__(moe) vars(moe).update( tp_size=4, + reduce_results=True, is_deepseek_v4=True, _shared_expert_tp1=shared_tp1, layer_id=0, is_nextn=False, is_hash=False, _fuse_shared_experts_inside_sbo=False, - _fuse_finalize_all_reduce=False, num_fused_shared_experts=0, routed_scaling_factor=1.0, experts=experts, @@ -233,7 +244,7 @@ def _moe_stub(rank, *, dual, shared_tp1): _forward_shared_experts=lambda x, *a, **kw: x * 0.5, forward=lambda x, *a, **kw: ( moe.forward_normal_dual_stream if dual else moe.forward_normal - )(x), + )(x, return_moe_output=kw.get("return_moe_output", False)), ) return moe @@ -282,6 +293,7 @@ def test_moe_model_handoff(group, rows, dual, defer, shared_tp1): "sglang.srt.models.deepseek_v2.post_experts_all_reduce", side_effect=group.all_reduce, ) as original_reduce, + patch("sglang.srt.layers.moe.post_experts_all_reduce", original_reduce), patch( "sglang.srt.layers.moe.get_moe_a2a_backend", return_value=MoeA2ABackend.NONE ), @@ -316,22 +328,24 @@ def test_moe_model_handoff(group, rows, dual, defer, shared_tp1): @pytest.mark.parametrize("flag", ["mlp_reduce_scatter"]) @pytest.mark.parametrize("dual", [False, True], ids=["normal", "dual-stream"]) -def test_moe_skipped_reduction_keeps_post(group, dual, flag): +def test_moe_skipped_reduction_declines_fusion(group, dual, flag): """A reduce-scattered MoE output must not reach the fused all-reduce + mHC - post: the pending post stays pending and no collective runs.""" - from sglang.srt.layers.moe.mhc_post_fusion import ( - MhcPostFusion, - use_mhc_post_fusion, + post: moe_mhc_fusion declines, so the MoE is never asked to defer, and its + own path runs no collective either.""" + from sglang.srt.layers.moe import MoeA2ABackend + from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( + moe_mhc_fusion, ) moe = _moe_stub(0, dual=dual, shared_tp1=False) + layer = SimpleNamespace(mlp=moe) x = torch.ones(1, 5120, device="cuda", dtype=torch.bfloat16) residual = torch.randn(1, 4, 5120, device="cuda", dtype=torch.bfloat16) - post = torch.rand(1, 4, device="cuda") - state = MhcPostFusion(residual, post, torch.rand(1, 4, 4, device="cuda"), None) with ( get_forward().scoped(**{flag: True}, flashinfer_trtllm_bypass=False), - use_mhc_post_fusion(state), + patch(f"{_FUSED_MHC}._can_fuse_mhc", lambda *_: True), + patch(f"{_FUSED_MHC}._make_mhc_fusion") as make_fusion, + patch(f"{_FUSED_MHC}.get_moe_a2a_backend", return_value=MoeA2ABackend.NONE), patch( "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce" ) as reduction, @@ -343,8 +357,9 @@ def test_moe_skipped_reduction_keeps_post(group, dual, flag): return_value=SimpleNamespace(moe=SimpleNamespace(enable_eplb=False)), ), ): + assert moe_mhc_fusion(layer, residual, None, None) is None actual = moe(x) - assert state.output is None + make_fusion.assert_not_called() reduction.assert_not_called() moe_reduction.assert_not_called() torch.testing.assert_close(actual, x * 1.5, atol=0, rtol=0)