[DeepSeek-V4.1] Unify mHC into one state machine and drop medium-batch fusion variants (#42245)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: BBuf <1182563586@qq.com>
This commit is contained in:
co-authored by
Claude Fable 5
BBuf
parent
2a5f45d3d4
commit
730f1f3e5b
@@ -510,7 +510,7 @@ struct MoeFinalizeAllReduceKernel {
|
||||
}
|
||||
|
||||
/// Finalize + all-reduce + HC=4 post; original reduced output is retained.
|
||||
static void run_mhc(
|
||||
static void run_mhc_post(
|
||||
CommunicatorRef ref,
|
||||
TensorView out,
|
||||
TensorView gemm2_out,
|
||||
@@ -538,41 +538,7 @@ struct MoeFinalizeAllReduceKernel {
|
||||
comb);
|
||||
}
|
||||
|
||||
// Keep the original RMSNorm as a separate kernel, while reusing the
|
||||
// BF16-rounded post values for the next sublayer's pre-combine.
|
||||
static void run_mhc_combine(
|
||||
CommunicatorRef ref,
|
||||
TensorView out,
|
||||
TensorView gemm2_out,
|
||||
TensorView permuted_idx,
|
||||
TensorView expert_weights,
|
||||
std::optional<TensorView> shared_output,
|
||||
TensorView mhc_out,
|
||||
TensorView residual,
|
||||
TensorView post,
|
||||
TensorView comb,
|
||||
TensorView pre,
|
||||
TensorView combined) {
|
||||
static_assert(kMhc && kCollapse && !kQuant);
|
||||
run_impl(
|
||||
ref,
|
||||
out,
|
||||
gemm2_out,
|
||||
permuted_idx,
|
||||
expert_weights,
|
||||
shared_output,
|
||||
std::nullopt,
|
||||
0.0,
|
||||
false,
|
||||
mhc_out,
|
||||
residual,
|
||||
post,
|
||||
comb,
|
||||
pre,
|
||||
combined);
|
||||
}
|
||||
|
||||
static void run_mhc_norm(
|
||||
static void run_mhc_post_combine_norm(
|
||||
CommunicatorRef ref,
|
||||
TensorView out,
|
||||
TensorView gemm2_out,
|
||||
@@ -606,7 +572,7 @@ struct MoeFinalizeAllReduceKernel {
|
||||
normalized);
|
||||
}
|
||||
|
||||
static void run_mhc_quant(
|
||||
static void run_mhc_post_combine_norm_quant(
|
||||
CommunicatorRef ref,
|
||||
TensorView out,
|
||||
TensorView gemm2_out,
|
||||
@@ -707,7 +673,7 @@ struct MoeFinalizeAllReduceKernel {
|
||||
}
|
||||
const auto num_tokens = static_cast<uint32_t>(T.unwrap());
|
||||
if constexpr (kQuant) {
|
||||
CHECK_HOST(num_tokens <= 8);
|
||||
CHECK_HOST(num_tokens <= 128) << "the quant epilogue writes one 128-row scale tile";
|
||||
CHECK_HOST(norm_weight.has_value());
|
||||
TensorMatcher({T, kHiddenDim}).with_dtype<fp8_e4m3_t>().with_device<kDLCUDA>(device).verify(quantized.value());
|
||||
TensorMatcher({(kHiddenDim / 32) * 128})
|
||||
|
||||
@@ -34,18 +34,21 @@ def _jit_mhc_module(world_size, top_k, cluster_size, weight_dtype):
|
||||
True,
|
||||
)
|
||||
return load_jit(
|
||||
"moe_finalize_all_reduce_mhc",
|
||||
"moe_finalize_all_reduce_mhc_post",
|
||||
*args,
|
||||
cuda_files=["distributed/all_reduce_fusion.cuh"],
|
||||
cuda_wrappers=[
|
||||
("run", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc"),
|
||||
("run_norm", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_norm"),
|
||||
("run", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_post"),
|
||||
(
|
||||
"run_norm",
|
||||
f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_post_combine_norm",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["out", "mhc_out"])
|
||||
def _moe_finalize_all_reduce_mhc_op(
|
||||
def _moe_finalize_all_reduce_mhc_post_op(
|
||||
world_size: int,
|
||||
top_k: int,
|
||||
cluster_size: int,
|
||||
@@ -75,7 +78,7 @@ def _moe_finalize_all_reduce_mhc_op(
|
||||
)
|
||||
|
||||
|
||||
def moe_finalize_all_reduce_mhc(
|
||||
def moe_finalize_all_reduce_mhc_post(
|
||||
gemm2: torch.Tensor,
|
||||
idx: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
@@ -99,7 +102,7 @@ def moe_finalize_all_reduce_mhc(
|
||||
)
|
||||
mhc_out = torch.empty_like(residual)
|
||||
if weights.shape[0]:
|
||||
_moe_finalize_all_reduce_mhc_op(
|
||||
_moe_finalize_all_reduce_mhc_post_op(
|
||||
world_size,
|
||||
top_k,
|
||||
cluster_size or default_cluster_size(_MHC_HIDDEN_DIM),
|
||||
@@ -117,7 +120,7 @@ def moe_finalize_all_reduce_mhc(
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["out", "mhc_out", "normalized"])
|
||||
def _moe_finalize_all_reduce_mhc_norm_op(
|
||||
def _moe_finalize_all_reduce_mhc_post_combine_norm_op(
|
||||
world_size: int,
|
||||
top_k: int,
|
||||
cluster_size: int,
|
||||
@@ -155,7 +158,7 @@ def _moe_finalize_all_reduce_mhc_norm_op(
|
||||
)
|
||||
|
||||
|
||||
def moe_finalize_all_reduce_mhc_norm(
|
||||
def moe_finalize_all_reduce_mhc_post_combine_norm(
|
||||
gemm2: torch.Tensor,
|
||||
idx: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
@@ -171,7 +174,7 @@ def moe_finalize_all_reduce_mhc_norm(
|
||||
world_size: int,
|
||||
cluster_size: Optional[int] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
""":func:`moe_finalize_all_reduce_mhc` plus the HC=4 pre-collapse
|
||||
""":func:`moe_finalize_all_reduce_mhc_post` plus the HC=4 pre-collapse
|
||||
(``pre`` ``[T, 4]`` fp32) and RMSNorm of the collapsed row. Returns
|
||||
``(reduced, mhc_out, normalized [T, hidden])``.
|
||||
"""
|
||||
@@ -181,7 +184,7 @@ def moe_finalize_all_reduce_mhc_norm(
|
||||
mhc_out = torch.empty_like(residual)
|
||||
normalized = torch.empty_like(out)
|
||||
if weights.shape[0]:
|
||||
_moe_finalize_all_reduce_mhc_norm_op(
|
||||
_moe_finalize_all_reduce_mhc_post_combine_norm_op(
|
||||
world_size,
|
||||
top_k,
|
||||
cluster_size or default_cluster_size(_MHC_HIDDEN_DIM),
|
||||
@@ -210,7 +213,7 @@ def _identity_routing(rows, device):
|
||||
)
|
||||
|
||||
|
||||
def all_reduce_mhc_norm(
|
||||
def all_reduce_mhc_post_combine_norm(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post: torch.Tensor,
|
||||
@@ -224,7 +227,7 @@ def all_reduce_mhc_norm(
|
||||
"""Plain all-reduce of ``x`` with the mHC + norm epilogue: the finalize
|
||||
kernel driven with identity routing (top_k = 1, unit weights)."""
|
||||
idx, weights = _identity_routing(x.shape[0], x.device)
|
||||
return moe_finalize_all_reduce_mhc_norm(
|
||||
return moe_finalize_all_reduce_mhc_post_combine_norm(
|
||||
x,
|
||||
idx,
|
||||
weights,
|
||||
@@ -254,17 +257,22 @@ def _jit_mhc_quant_module(world_size, top_k, cluster_size, weight_dtype):
|
||||
True,
|
||||
)
|
||||
return load_jit(
|
||||
"moe_finalize_all_reduce_mhc_quant",
|
||||
"moe_finalize_all_reduce_mhc_post_combine_norm_quant",
|
||||
*args,
|
||||
cuda_files=["distributed/all_reduce_fusion.cuh"],
|
||||
cuda_wrappers=[("run", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_quant")],
|
||||
cuda_wrappers=[
|
||||
(
|
||||
"run",
|
||||
f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_post_combine_norm_quant",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@register_custom_op(
|
||||
mutates_args=["out", "mhc_out", "normalized", "quantized", "scales"]
|
||||
)
|
||||
def _moe_finalize_all_reduce_mhc_quant_op(
|
||||
def _moe_finalize_all_reduce_mhc_post_combine_norm_quant_op(
|
||||
world_size: int,
|
||||
top_k: int,
|
||||
cluster_size: int,
|
||||
@@ -306,7 +314,7 @@ def _moe_finalize_all_reduce_mhc_quant_op(
|
||||
)
|
||||
|
||||
|
||||
def moe_finalize_all_reduce_mhc_quant(
|
||||
def moe_finalize_all_reduce_mhc_post_combine_norm_quant(
|
||||
gemm2: torch.Tensor,
|
||||
idx: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
@@ -322,12 +330,14 @@ def moe_finalize_all_reduce_mhc_quant(
|
||||
world_size: int,
|
||||
cluster_size: Optional[int] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
""":func:`moe_finalize_all_reduce_mhc_norm` plus fp8 e4m3 quantization of
|
||||
the normalized row with ue8m0 group scales (rows <= 8). Returns
|
||||
""":func:`moe_finalize_all_reduce_mhc_post_combine_norm` plus fp8 e4m3 quantization of
|
||||
the normalized row with ue8m0 group scales (rows <= 128: the scale swizzle
|
||||
addresses a single 128-row tile). Returns
|
||||
``(reduced, mhc_out, normalized, quantized, scales)``.
|
||||
"""
|
||||
rows = weights.shape[0]
|
||||
assert 0 < rows <= 8
|
||||
# One 128-row scale tile; larger needs a (token/128) block offset in the kernel.
|
||||
assert 0 < rows <= 128
|
||||
out = torch.empty(
|
||||
(rows, _MHC_HIDDEN_DIM), dtype=torch.bfloat16, device=gemm2.device
|
||||
)
|
||||
@@ -338,7 +348,7 @@ def moe_finalize_all_reduce_mhc_quant(
|
||||
scales = torch.empty(
|
||||
(_MHC_HIDDEN_DIM // 32) * 128, device=gemm2.device, dtype=torch.uint8
|
||||
)
|
||||
_moe_finalize_all_reduce_mhc_quant_op(
|
||||
_moe_finalize_all_reduce_mhc_post_combine_norm_quant_op(
|
||||
world_size,
|
||||
top_k,
|
||||
cluster_size or default_cluster_size(_MHC_HIDDEN_DIM),
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
"""Finalize/all-reduce + post + combine, retaining standalone RMSNorm."""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
is_arch_support_pdl,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
from sglang.kernels.ops.communication.all_reduce_fusion import (
|
||||
default_cluster_size,
|
||||
get_registered_comm,
|
||||
)
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
|
||||
@cache_once
|
||||
def _module(world_size, top_k, cluster_size, weight_dtype):
|
||||
args = make_cpp_args(
|
||||
world_size,
|
||||
5120,
|
||||
top_k,
|
||||
cluster_size,
|
||||
is_arch_support_pdl(),
|
||||
weight_dtype,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
)
|
||||
return load_jit(
|
||||
"moe_finalize_all_reduce_mhc_combine",
|
||||
*args,
|
||||
cuda_files=["distributed/all_reduce_fusion.cuh"],
|
||||
cuda_wrappers=[("run", f"MoeFinalizeAllReduceKernel<{args}>::run_mhc_combine")],
|
||||
)
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["out", "mhc_out", "combined"])
|
||||
def _moe_finalize_all_reduce_mhc_combine(
|
||||
world_size: int,
|
||||
top_k: int,
|
||||
cluster_size: int,
|
||||
out: torch.Tensor,
|
||||
mhc_out: torch.Tensor,
|
||||
combined: torch.Tensor,
|
||||
gemm2: torch.Tensor,
|
||||
idx: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
shared: Optional[torch.Tensor],
|
||||
residual: torch.Tensor,
|
||||
post: torch.Tensor,
|
||||
comb: torch.Tensor,
|
||||
pre: torch.Tensor,
|
||||
) -> None:
|
||||
comm = get_registered_comm(world_size)
|
||||
assert comm is not None
|
||||
_module(world_size, top_k, cluster_size, weights.dtype).run(
|
||||
comm,
|
||||
out,
|
||||
gemm2,
|
||||
idx,
|
||||
weights,
|
||||
shared,
|
||||
mhc_out,
|
||||
residual,
|
||||
post,
|
||||
comb,
|
||||
pre,
|
||||
combined,
|
||||
)
|
||||
|
||||
|
||||
def moe_finalize_all_reduce_mhc_combine(
|
||||
gemm2,
|
||||
idx,
|
||||
weights,
|
||||
top_k,
|
||||
shared,
|
||||
residual,
|
||||
post,
|
||||
comb,
|
||||
pre,
|
||||
*,
|
||||
world_size,
|
||||
cluster_size=None,
|
||||
):
|
||||
out = torch.empty(
|
||||
(weights.shape[0], 5120), dtype=torch.bfloat16, device=gemm2.device
|
||||
)
|
||||
mhc_out = torch.empty_like(residual)
|
||||
combined = torch.empty_like(out)
|
||||
if weights.shape[0]:
|
||||
_moe_finalize_all_reduce_mhc_combine(
|
||||
world_size,
|
||||
top_k,
|
||||
cluster_size or default_cluster_size(5120),
|
||||
out,
|
||||
mhc_out,
|
||||
combined,
|
||||
gemm2,
|
||||
idx,
|
||||
weights,
|
||||
shared,
|
||||
residual,
|
||||
post,
|
||||
comb,
|
||||
pre,
|
||||
)
|
||||
return out, mhc_out, combined
|
||||
|
||||
|
||||
def all_reduce_mhc_combine(x, residual, post, comb, pre, *, world_size):
|
||||
from sglang.kernels.ops.communication.all_reduce_mhc import _identity_routing
|
||||
|
||||
idx, weights = _identity_routing(x.shape[0], x.device)
|
||||
return moe_finalize_all_reduce_mhc_combine(
|
||||
x, idx, weights, 1, None, residual, post, comb, pre, world_size=world_size
|
||||
)
|
||||
@@ -55,9 +55,12 @@ def _hc_combine_norm_prefill(
|
||||
def hc_combine_norm(
|
||||
x: torch.Tensor, pre: torch.Tensor, weight: torch.Tensor, eps: float
|
||||
) -> torch.Tensor:
|
||||
"""Fuse four-stream combine and RMSNorm for BF16 batches of width 5120."""
|
||||
"""Fuse four-stream combine and RMSNorm for BF16 batches of width 5120.
|
||||
|
||||
Every row count takes the same per-row reduction in the same order, so the
|
||||
result is batch-invariant across the grid-split variants below."""
|
||||
m = x.shape[0]
|
||||
assert (0 < m <= 96 or 4096 <= m <= 65536) and x.shape == (m, 20480)
|
||||
assert x.shape == (m, 20480)
|
||||
assert pre.shape == (m, 4) and pre.stride(1) == 1
|
||||
assert weight.shape == (5120,) and weight.is_contiguous()
|
||||
assert x.dtype == weight.dtype == torch.bfloat16 and x.stride(1) == 1
|
||||
@@ -133,7 +136,7 @@ def hc_combine_norm_mxfp8(
|
||||
):
|
||||
"""Four-stream combine + RMSNorm returning ``(y_bf16, y_q, y_sf)``."""
|
||||
m = x.shape[0]
|
||||
assert 0 < m <= 8, "the fused MXFP8 epilogue only supports small decode/verify"
|
||||
assert m <= 128, "the MXFP8 scale swizzle addresses a single 128-row tile"
|
||||
k = x.shape[1] // 4
|
||||
y = torch.empty((m, k), dtype=x.dtype, device=x.device)
|
||||
q, s = _alloc(m, k, x.device)
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
"""HC=4 post-mix and pre-combine with the original BF16 intermediates."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mhc_post_combine(X, R, P, C, A, RO, Y, H: tl.constexpr, B: tl.constexpr):
|
||||
row = tl.program_id(0)
|
||||
h = tl.program_id(1) * B + tl.arange(0, B)
|
||||
mask = h < H
|
||||
x = tl.load(X + row * H + h, mask, 0).to(tl.float32)
|
||||
r0 = tl.load(R + (row * 4 + 0) * H + h, mask, 0).to(tl.float32)
|
||||
r1 = tl.load(R + (row * 4 + 1) * H + h, mask, 0).to(tl.float32)
|
||||
r2 = tl.load(R + (row * 4 + 2) * H + h, mask, 0).to(tl.float32)
|
||||
r3 = tl.load(R + (row * 4 + 3) * H + h, mask, 0).to(tl.float32)
|
||||
collapsed = tl.full((B,), 0, tl.float32)
|
||||
for i in tl.static_range(4):
|
||||
post = tl.load(P + row * 4 + i)
|
||||
c0 = tl.load(C + row * 16 + i)
|
||||
c1 = tl.load(C + row * 16 + 4 + i)
|
||||
c2 = tl.load(C + row * 16 + 8 + i)
|
||||
c3 = tl.load(C + row * 16 + 12 + i)
|
||||
pre = tl.load(A + row * 4 + i)
|
||||
# Match mhc_post_split_h's contraction order and BF16 store.
|
||||
mixed = tl.fma(post, x, c0 * r0)
|
||||
mixed = tl.fma(c1, r1, mixed)
|
||||
mixed = tl.fma(c2, r2, mixed)
|
||||
mixed = tl.fma(c3, r3, mixed)
|
||||
rounded = mixed.to(tl.bfloat16)
|
||||
tl.store(RO + (row * 4 + i) * H + h, rounded, mask)
|
||||
# Match hc_combine's sequential FP32 accumulation, then BF16 store.
|
||||
collapsed = tl.fma(pre, rounded.to(tl.float32), collapsed)
|
||||
tl.store(Y + row * H + h, collapsed, mask)
|
||||
|
||||
|
||||
def mhc_post_combine(x, residual, post, comb, pre):
|
||||
"""Return (updated HC streams, collapsed BF16 input) out of place."""
|
||||
assert x.dtype == residual.dtype == torch.bfloat16
|
||||
assert post.dtype == comb.dtype == pre.dtype == torch.float32
|
||||
assert x.ndim == 2 and x.shape[1] == 5120
|
||||
assert residual.shape == (x.shape[0], 4, x.shape[1])
|
||||
assert post.shape == pre.shape == (x.shape[0], 4)
|
||||
assert comb.shape == (x.shape[0], 4, 4)
|
||||
assert all(t.is_contiguous() for t in (x, residual, post, comb, pre))
|
||||
updated = torch.empty_like(residual)
|
||||
combined = torch.empty_like(x)
|
||||
if x.shape[0]:
|
||||
block = 1024 if x.shape[0] <= 192 or x.shape[0] >= 4096 else 512
|
||||
_mhc_post_combine[(x.shape[0], triton.cdiv(x.shape[1], block))](
|
||||
x,
|
||||
residual,
|
||||
post,
|
||||
comb,
|
||||
pre,
|
||||
updated,
|
||||
combined,
|
||||
H=x.shape[1],
|
||||
B=block,
|
||||
num_warps=4,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return updated, combined
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_norm_prefill(X, W, Y, EPS: tl.constexpr):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
h = tl.arange(0, 8192)
|
||||
value = tl.load(X + row * 5120 + h, h < 5120, 0).to(tl.float32)
|
||||
inv_rms = tl.rsqrt(tl.sum(value * value, 0) / 5120 + EPS)
|
||||
weight = tl.load(W + h, h < 5120, 0).to(tl.float32)
|
||||
tl.store(Y + row * 5120 + h, value * inv_rms * weight, h < 5120)
|
||||
|
||||
|
||||
def hc_norm_prefill(combined, weight, eps):
|
||||
"""Keep the original fused prefill norm's arithmetic on collapsed input."""
|
||||
assert 4096 <= combined.shape[0] <= 65536 and combined.shape[1] == 5120
|
||||
assert weight.shape == (5120,)
|
||||
assert combined.dtype == weight.dtype == torch.bfloat16
|
||||
assert combined.is_contiguous() and weight.is_contiguous()
|
||||
output = torch.empty_like(combined)
|
||||
_hc_norm_prefill[(combined.shape[0],)](combined, weight, output, eps, num_warps=4)
|
||||
return output
|
||||
@@ -1,57 +0,0 @@
|
||||
"""Scoped handoff of a decoder's HC post operands to deferred MoE finalize."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
@dataclass
|
||||
class MhcPostFusion:
|
||||
residual: torch.Tensor
|
||||
post: Optional[torch.Tensor]
|
||||
comb: Optional[torch.Tensor]
|
||||
stats_stream: Optional[torch.cuda.Stream]
|
||||
output: Optional[torch.Tensor] = None
|
||||
pre: Optional[torch.Tensor] = None
|
||||
norm_weight: Optional[torch.Tensor] = None
|
||||
norm_eps: float = 0.0
|
||||
normalized: Optional[torch.Tensor] = None
|
||||
quantized: Optional[tuple[torch.Tensor, torch.Tensor]] = None
|
||||
combine_only: bool = False
|
||||
combined: Optional[torch.Tensor] = None
|
||||
overlap_only: bool = False
|
||||
record_stats: Optional[
|
||||
Callable[[], tuple[torch.Tensor, torch.Tensor, torch.Tensor]]
|
||||
] = None
|
||||
|
||||
def start_stats_before_all_reduce(self):
|
||||
if self.overlap_only and self.record_stats is not None:
|
||||
assert self.stats_stream is not None
|
||||
self.stats_stream.wait_stream(torch.cuda.current_stream())
|
||||
self.materialize_stats()
|
||||
|
||||
def materialize_stats(self):
|
||||
# Record after the main parent; graph replay must keep the join on the
|
||||
# main stream.
|
||||
if self.record_stats is not None:
|
||||
self.pre, self.post, self.comb = self.record_stats()
|
||||
self.record_stats = None
|
||||
|
||||
|
||||
_current: ContextVar[Optional[MhcPostFusion]] = ContextVar("moe_mhc_post", default=None)
|
||||
|
||||
|
||||
def current_mhc_post_fusion():
|
||||
return _current.get()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def use_mhc_post_fusion(state):
|
||||
token = _current.set(state)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_current.reset(token)
|
||||
@@ -81,7 +81,7 @@ def is_deferred_finalize_enabled() -> bool:
|
||||
|
||||
def finalize_flashinfer_trtllm_deferred_output(
|
||||
deferred_output: FlashInferTrtllmDeferredFinalizeOutput,
|
||||
shared_output: torch.Tensor,
|
||||
shared_output: Optional[torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
from sglang.kernels.ops.moe.moe_finalize_fuse_shared import moe_finalize_fuse_shared
|
||||
|
||||
|
||||
@@ -519,82 +519,43 @@ def maybe_fuse_routed_scale_and_shared_add(
|
||||
return routed
|
||||
|
||||
|
||||
# Fused finalize + shared add + TP all-reduce
|
||||
_fused_finalize_all_reduce_world_size: Optional[int] = None
|
||||
_fused_finalize_all_reduce_probed = False
|
||||
_fused_finalize_all_reduce_comm = None
|
||||
# Fused finalize + shared add + TP all-reduce, staged through the TP group's own
|
||||
# CustomAllReduceV2 push plane (tiny batches only; no dedicated workspace).
|
||||
|
||||
|
||||
def _fused_finalize_all_reduce_comm_world_size() -> Optional[int]:
|
||||
"""Reserve a separate push plane for up to 384 rows of fused MoE output."""
|
||||
global _fused_finalize_all_reduce_world_size, _fused_finalize_all_reduce_probed
|
||||
global _fused_finalize_all_reduce_comm
|
||||
if not _fused_finalize_all_reduce_probed:
|
||||
# The eager warmup must initialize peer workspaces before capture.
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
return None
|
||||
_fused_finalize_all_reduce_probed = True
|
||||
from sglang.kernels.ops.communication import all_reduce_fusion
|
||||
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
|
||||
CustomAllReduceV2,
|
||||
def register_fused_all_reduce_comm() -> None:
|
||||
"""Hand the TP group's push plane to the fusion kernels, once per process.
|
||||
Must run before graph capture; unregistered means everything runs unfused."""
|
||||
from sglang.kernels.ops.communication import all_reduce_fusion
|
||||
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
|
||||
CustomAllReduceV2,
|
||||
)
|
||||
|
||||
ca_comm = get_parallel().tp_group.ca_comm
|
||||
if isinstance(ca_comm, CustomAllReduceV2) and not ca_comm.disabled:
|
||||
all_reduce_fusion.register_comm(ca_comm.obj)
|
||||
else:
|
||||
log_info_on_rank0(
|
||||
logger,
|
||||
"Fused MoE finalize: TP group has no "
|
||||
"CustomAllReduceV2 push plane; keeping the unfused finalize path",
|
||||
)
|
||||
|
||||
ca_comm = get_parallel().tp_group.ca_comm
|
||||
if isinstance(ca_comm, CustomAllReduceV2) and not ca_comm.disabled:
|
||||
fused_comm = ca_comm
|
||||
if ca_comm.world_size == 4:
|
||||
from sglang.kernels.ops.communication.mp import register_comm_cleanup
|
||||
|
||||
fused_comm = CustomAllReduceV2(
|
||||
ca_comm.group,
|
||||
ca_comm.device,
|
||||
max_pull_size=0,
|
||||
max_pull_blocks=0,
|
||||
max_push_size=4 * 1024 * 1024,
|
||||
max_push_blocks=512,
|
||||
)
|
||||
register_comm_cleanup(fused_comm)
|
||||
if fused_comm.disabled:
|
||||
return None
|
||||
_fused_finalize_all_reduce_comm = fused_comm
|
||||
all_reduce_fusion.register_comm(fused_comm.obj)
|
||||
_fused_finalize_all_reduce_world_size = fused_comm.world_size
|
||||
else:
|
||||
log_info_on_rank0(
|
||||
logger,
|
||||
"Fused MoE finalize: TP group has no "
|
||||
"CustomAllReduceV2 push plane; keeping the unfused finalize path",
|
||||
)
|
||||
return _fused_finalize_all_reduce_world_size
|
||||
def can_fuse_all_reduce(num_tokens: int, hidden_dim: int) -> bool:
|
||||
"""Whether the registered push plane can stage ``[num_tokens, hidden_dim]``
|
||||
BF16 rows: one slot holds them and each row owns a push phase counter. The
|
||||
384 ceiling is the bench-validated range of the fused collective family.
|
||||
|
||||
|
||||
def should_use_fuse_finalize_all_reduce(
|
||||
experts, num_tokens: int, hidden_dim: int
|
||||
) -> bool:
|
||||
"""Capability only; the batch-size policy cap lives at the call site. The
|
||||
kernel never rescales, so the expert weights must carry the routed scaling."""
|
||||
if not isinstance(experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod):
|
||||
return False
|
||||
if experts.quant_method.flashinfer_mxfp4_moe_precision != "default":
|
||||
return False
|
||||
if not experts.should_fuse_routed_scaling_factor_in_topk:
|
||||
return False
|
||||
if num_tokens <= 0:
|
||||
return False
|
||||
if num_tokens > 96:
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
|
||||
if is_batch_invariant_mode_enabled():
|
||||
return False
|
||||
The deferred-finalize handle type carries its own numeric contract (every
|
||||
producer folds the routed scaling; no consumer rescales)."""
|
||||
from sglang.kernels.ops.communication import all_reduce_fusion
|
||||
|
||||
if not all_reduce_fusion.valid_cluster_sizes(hidden_dim):
|
||||
if all_reduce_fusion.get_registered_comm(get_parallel().tp_size) is None:
|
||||
return False
|
||||
tp_group = get_parallel().tp_group
|
||||
if _fused_finalize_all_reduce_comm_world_size() != tp_group.world_size:
|
||||
return False
|
||||
comm = _fused_finalize_all_reduce_comm
|
||||
# Each token row owns a push phase counter on the epilogue's plane.
|
||||
if num_tokens > comm.config.num_push_blocks:
|
||||
return False
|
||||
return all_reduce_fusion.fits_push_slot(comm.max_push_size, num_tokens, hidden_dim)
|
||||
comm = get_parallel().tp_group.ca_comm
|
||||
return (
|
||||
num_tokens <= 384
|
||||
and num_tokens <= comm.config.num_push_blocks
|
||||
and all_reduce_fusion.fits_push_slot(comm.max_push_size, num_tokens, hidden_dim)
|
||||
)
|
||||
|
||||
@@ -7,13 +7,7 @@ from typing import Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.communication.all_reduce_mhc_hip import all_reduce_mhc_post
|
||||
from sglang.srt.layers.moe.mhc_post_fusion import MhcPostFusion
|
||||
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import (
|
||||
ALL_REDUCE_MHC_MAX_ROWS,
|
||||
)
|
||||
from sglang.srt.models.deepseek_common.utils import _use_aiter
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
if _use_aiter:
|
||||
from sglang.kernels.ops.moe.rocm_router_gate import rocm_router_max_tokens
|
||||
@@ -58,27 +52,3 @@ def forward_gate(
|
||||
if logits_and_partials is not None:
|
||||
return logits_and_partials
|
||||
return moe.gate(hidden_states, gemm_output_zero_allocator), None
|
||||
|
||||
|
||||
def fused_all_reduce_mhc(
|
||||
moe, mhc: Optional[MhcPostFusion], hidden_states: torch.Tensor
|
||||
) -> bool:
|
||||
"""Whether the post-experts all-reduce of moe (a DeepseekV2MoE) ran fused with the
|
||||
mHC post, for the eagerly built 1-8 row mHC states; the decoder then reads mhc.output
|
||||
and hidden_states only keeps the DP wrapper's tensor contract."""
|
||||
if not (
|
||||
mhc is not None
|
||||
and not moe._shared_expert_tp1
|
||||
and not mhc.overlap_only
|
||||
and mhc.post is not None
|
||||
and 1 <= hidden_states.shape[0] <= ALL_REDUCE_MHC_MAX_ROWS
|
||||
):
|
||||
return False
|
||||
mhc.output = all_reduce_mhc_post(
|
||||
hidden_states,
|
||||
mhc.residual,
|
||||
mhc.post,
|
||||
mhc.comb,
|
||||
get_parallel().tp_group.ca_comm,
|
||||
)
|
||||
return True
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import triton
|
||||
@@ -10,7 +10,9 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.layers.moe.mhc_post_fusion import MhcPostFusion, use_mhc_post_fusion
|
||||
from sglang.srt.layers.moe.utils import should_skip_post_experts_all_reduce
|
||||
from sglang.srt.models.deepseek_v2 import MoEOutput
|
||||
from sglang.srt.models.deepseek_v4_mhc import AttnOutput
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_forward,
|
||||
@@ -38,6 +40,19 @@ if _is_gfx95_supported:
|
||||
post_attention_norm,
|
||||
)
|
||||
|
||||
|
||||
class MhcPostOperands(NamedTuple):
|
||||
"""What the fused all-reduce + hc_post kernel reads besides the sublayer rows.
|
||||
|
||||
The caller hands this to `apply_attention_mhc` / `apply_moe_mhc` directly; the
|
||||
sublayer only has to be told to leave its reduce alone.
|
||||
"""
|
||||
|
||||
residual: torch.Tensor
|
||||
post: torch.Tensor
|
||||
comb: torch.Tensor
|
||||
|
||||
|
||||
# the fused all-reduce + hc_post kernel: DeepSeek-V4.1's hidden size, up to this many rows
|
||||
MHC_HIDDEN_SIZE = 5120
|
||||
ALL_REDUCE_MHC_MAX_ROWS = 8
|
||||
@@ -422,7 +437,7 @@ def apply_mhc_post_pre_boundary(
|
||||
if not _is_fused_mhc_post_pre_enabled():
|
||||
return None
|
||||
|
||||
from sglang.srt.models.deepseek_v4 import _get_mhc_ops
|
||||
from sglang.srt.models.deepseek_v4 import get_mhc_ops
|
||||
|
||||
post_in = post.unsqueeze(-1) if post.ndim == 2 else post
|
||||
(
|
||||
@@ -430,7 +445,7 @@ def apply_mhc_post_pre_boundary(
|
||||
post_out,
|
||||
comb_out,
|
||||
layer_input_out,
|
||||
) = _get_mhc_ops().mhc_fused_post_pre(
|
||||
) = get_mhc_ops().mhc_fused_post_pre(
|
||||
layer_input,
|
||||
residual,
|
||||
post_in,
|
||||
@@ -508,14 +523,12 @@ def _can_fuse_mhc(layer, residual: torch.Tensor, forward_batch) -> bool:
|
||||
|
||||
def _make_mhc_fusion(
|
||||
residual: torch.Tensor, coefficients: "HcCoefficients", comm
|
||||
) -> Optional[MhcPostFusion]:
|
||||
) -> Optional[MhcPostOperands]:
|
||||
# comm is the group's aiter CustomAllreduce, or None when custom all-reduce is off
|
||||
if comm is None or comm.disabled or not comm.enable_register_for_capturing:
|
||||
return None
|
||||
coefficients.materialize()
|
||||
return MhcPostFusion(
|
||||
residual, coefficients.post, coefficients.comb, None, pre=coefficients.pre
|
||||
)
|
||||
return MhcPostOperands(residual, coefficients.post, coefficients.comb)
|
||||
|
||||
|
||||
def attention_mhc_fusion(layer, residual, coefficients, forward_batch):
|
||||
@@ -536,21 +549,29 @@ def moe_mhc_fusion(layer, residual, coefficients, forward_batch):
|
||||
and layer.mlp.tp_size == 4
|
||||
and not layer.mlp._shared_expert_tp1
|
||||
and get_moe_a2a_backend().is_none()
|
||||
# A reduce some later step already owns, or one the combine performed, is
|
||||
# not ours to fuse into the post.
|
||||
and not should_skip_post_experts_all_reduce(is_tp_path=True)
|
||||
):
|
||||
return None
|
||||
return _make_mhc_fusion(residual, coefficients, get_parallel().tp_group.ca_comm)
|
||||
|
||||
|
||||
def apply_attention_mhc(x: torch.Tensor, state: MhcPostFusion) -> None:
|
||||
# a lazily recorded state still needs its stats before the fused kernel reads them
|
||||
state.materialize_stats()
|
||||
if state.stats_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(state.stats_stream)
|
||||
state.output = all_reduce_mhc_post(
|
||||
def apply_attention_mhc(x: torch.Tensor, state: MhcPostOperands) -> torch.Tensor:
|
||||
"""Attention's all-reduce fused with hc_post; returns the rebuilt streams."""
|
||||
return all_reduce_mhc_post(
|
||||
x, state.residual, state.post, state.comb, get_parallel().attn_tp_group.ca_comm
|
||||
)
|
||||
|
||||
|
||||
def apply_moe_mhc(x: torch.Tensor, state: MhcPostOperands) -> torch.Tensor:
|
||||
"""The same for the MoE's post-experts reduce, on rows it merged but left
|
||||
unreduced."""
|
||||
return all_reduce_mhc_post(
|
||||
x, state.residual, state.post, state.comb, get_parallel().tp_group.ca_comm
|
||||
)
|
||||
|
||||
|
||||
def forward_hc_pre_from_prev_fused_boundary(
|
||||
layer,
|
||||
positions: torch.Tensor,
|
||||
@@ -581,19 +602,25 @@ def forward_hc_pre_from_prev_fused_boundary(
|
||||
)
|
||||
with layer.self_attn.maybe_use_decode_attn_tp(forward_batch):
|
||||
mhc = attention_mhc_fusion(layer, residual, attn_coefficients, forward_batch)
|
||||
with use_mhc_post_fusion(mhc):
|
||||
x = layer.self_attn(
|
||||
x=x, positions=positions, forward_batch=forward_batch, x_quant=x_quant
|
||||
)
|
||||
x = layer.self_attn(
|
||||
x=x,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
x_quant=x_quant,
|
||||
defer_all_reduce=mhc is not None,
|
||||
)
|
||||
if mhc is not None:
|
||||
residual = mhc.output
|
||||
# wo_b honors the deferred reduce under attention_mhc_fusion's gates;
|
||||
# fail loud rather than fall back to an unfused reduce.
|
||||
assert isinstance(x, AttnOutput)
|
||||
residual = apply_attention_mhc(x.partial, mhc)
|
||||
x = None
|
||||
residual, x, ffn_coefficients = hc_boundary(
|
||||
layer,
|
||||
x,
|
||||
residual,
|
||||
attn_coefficients.post if mhc is None else None,
|
||||
attn_coefficients.comb if mhc is None else None,
|
||||
attn_coefficients.post if x is not None else None,
|
||||
attn_coefficients.comb if x is not None else None,
|
||||
attn_coefficients.pre,
|
||||
layer.hc_ffn_fn,
|
||||
layer.hc_ffn_scale,
|
||||
@@ -601,15 +628,22 @@ def forward_hc_pre_from_prev_fused_boundary(
|
||||
)
|
||||
x = _gfx95_dense_post_attention_norm(layer, x, ffn_coefficients)
|
||||
mhc = moe_mhc_fusion(layer, residual, ffn_coefficients, forward_batch)
|
||||
with use_mhc_post_fusion(mhc):
|
||||
x = layer._run_moe_ffn_dp_sync(
|
||||
x, forward_batch, input_ids=input_ids, input_ids_global=input_ids_global
|
||||
)
|
||||
x = layer._run_moe_ffn_dp_sync(
|
||||
x,
|
||||
forward_batch,
|
||||
input_ids=input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
return_moe_output=mhc is not None,
|
||||
)
|
||||
ffn_pre, ffn_post, ffn_comb = ffn_coefficients.tensors()
|
||||
if mhc is not None and mhc.output is not None:
|
||||
if mhc is not None:
|
||||
# The MoE TP path honors the deferred output under moe_mhc_fusion's gates;
|
||||
# fail loud rather than fall back to an unfused reduce. The fused post
|
||||
# carries the reduce, so take the merged rows without one.
|
||||
assert isinstance(x, MoEOutput)
|
||||
# Reduction already applied post. The next boundary consumes this
|
||||
# materialized residual, including when it would normally defer post.
|
||||
return mhc.output, ffn_pre, None
|
||||
return apply_moe_mhc(x.get_merged(), mhc), ffn_pre, None
|
||||
if defer_post:
|
||||
return None, ffn_pre, (x, residual, ffn_post, ffn_comb)
|
||||
return layer.hc_post(x, residual, ffn_post, ffn_comb), ffn_pre, None
|
||||
|
||||
@@ -167,20 +167,15 @@ def init_decoder_layer(layer, quant_config) -> None:
|
||||
)
|
||||
|
||||
|
||||
def hc_post(layer, x, residual, post, comb) -> Optional[torch.Tensor]:
|
||||
def hc_post(cfg, x, residual, post, comb) -> Optional[torch.Tensor]:
|
||||
"""V4.1's gfx950 hc_post at 768-4096 rows: the split-H kernel with 2048-wide blocks,
|
||||
whose ordinary stores preserve locality for the following mHC reader; None elsewhere."""
|
||||
if not (
|
||||
_is_gfx95_supported
|
||||
and layer.config.model_type == "deepseek_v41"
|
||||
and cfg.pre_from_prev
|
||||
and cfg.mult == 4
|
||||
and 768 <= x.shape[0] <= 4096
|
||||
and x.shape[1] == 5120
|
||||
and residual.shape == (x.shape[0], 4, 5120)
|
||||
and post.shape == (x.shape[0], 4)
|
||||
and comb.shape == (x.shape[0], 4, 4)
|
||||
and x.dtype == residual.dtype == torch.bfloat16
|
||||
and post.dtype == comb.dtype == torch.float32
|
||||
and all(t.is_contiguous() for t in (x, residual, post, comb))
|
||||
):
|
||||
return None
|
||||
return mhc_post_split_h(x, residual, post, comb, block_size=2048)
|
||||
@@ -216,14 +211,12 @@ def input_norm(
|
||||
def forward_layer_fused_boundary(
|
||||
model,
|
||||
i: int,
|
||||
state,
|
||||
*,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: Optional[torch.Tensor],
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch,
|
||||
input_ids_global: torch.Tensor,
|
||||
prev_pre: Optional[torch.Tensor],
|
||||
pending_post: Optional[Tuple[torch.Tensor, ...]],
|
||||
capture_dspark: bool,
|
||||
):
|
||||
"""Layer i through the fused mHC boundary. Its FFN hc_post stays pending for the next
|
||||
@@ -235,14 +228,23 @@ def forward_layer_fused_boundary(
|
||||
and model.layers[nxt].engram is None
|
||||
and not (capture_dspark and nxt in model.dspark_layers_to_capture)
|
||||
)
|
||||
return forward_hc_pre_from_prev_fused_boundary(
|
||||
# The boundary speaks its own (hidden_states, prev_pre, pending_post) triple;
|
||||
# the loop speaks HcState. This is the only place the two meet.
|
||||
from sglang.srt.models.deepseek_v4_mhc import HcPending, HcState
|
||||
|
||||
pending = isinstance(state.streams, HcPending)
|
||||
hidden_states, prev_pre, pending_post = forward_hc_pre_from_prev_fused_boundary(
|
||||
model.layers[i],
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
hidden_states=None if pending else state.streams,
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
input_ids_global=input_ids_global,
|
||||
prev_pre=prev_pre,
|
||||
pending_post=pending_post,
|
||||
prev_pre=state.pre,
|
||||
pending_post=tuple(state.streams) if pending else None,
|
||||
defer_post=defer_post,
|
||||
)
|
||||
return HcState(
|
||||
hidden_states if pending_post is None else HcPending(*pending_post),
|
||||
prev_pre,
|
||||
)
|
||||
|
||||
@@ -23,7 +23,17 @@ from __future__ import annotations
|
||||
import logging
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from functools import cached_property
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -96,7 +106,6 @@ from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
get_moe_runner_backend,
|
||||
post_experts_all_reduce,
|
||||
should_skip_post_experts_all_reduce,
|
||||
should_use_flashinfer_cutlass_moe_fp4_allgather,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
@@ -133,7 +142,6 @@ from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
|
||||
Mxfp8RoutedInputPreQuant,
|
||||
maybe_fuse_routed_scale_and_shared_add,
|
||||
routed_hidden_size,
|
||||
should_use_fuse_finalize_all_reduce,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope_wrapper
|
||||
@@ -582,9 +590,33 @@ class MoEGate(nn.Module):
|
||||
return logits
|
||||
|
||||
|
||||
# The dedicated 4 MiB push slot fits 384 rows of 5120 BF16 values.
|
||||
# The dispatch gate also checks slot capacity and available counters.
|
||||
_FUSED_FINALIZE_ALL_REDUCE_MAX_TOKENS = 384
|
||||
class MoEOutput(NamedTuple):
|
||||
"""A MoE block's pieces, before anything merges or reduces them. ``routed`` is
|
||||
a deferred finalize handle or the raw routed rows."""
|
||||
|
||||
routed: Any
|
||||
shared: Optional[torch.Tensor]
|
||||
experts: nn.Module
|
||||
routed_scaling_factor: float
|
||||
shared_is_replicated: bool
|
||||
|
||||
def get_merged(self) -> torch.Tensor:
|
||||
"""Merge the pieces, leaving the reduction to the caller."""
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
FlashInferTrtllmDeferredFinalizeOutput,
|
||||
finalize_flashinfer_trtllm_deferred_output,
|
||||
)
|
||||
|
||||
# NOTE: normal TP-sharded shared expert out
|
||||
shared = None if self.shared_is_replicated else self.shared
|
||||
if isinstance(self.routed, FlashInferTrtllmDeferredFinalizeOutput):
|
||||
return finalize_flashinfer_trtllm_deferred_output(self.routed, shared)
|
||||
return maybe_fuse_routed_scale_and_shared_add(
|
||||
self.experts,
|
||||
self.routed,
|
||||
shared,
|
||||
self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV2MoE(nn.Module):
|
||||
@@ -645,13 +677,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
self.routed_quant_stream = routed_quant_stream
|
||||
self.is_nextn = is_nextn
|
||||
self.is_deepseek_v4 = is_deepseek_v4
|
||||
self._fuse_finalize_all_reduce = (
|
||||
is_deepseek_v4
|
||||
and getattr(config, "hc_pre_from_prev_sublayer", False)
|
||||
and get_platform().is_blackwell
|
||||
and self.tp_size == 4
|
||||
)
|
||||
|
||||
n_hash_layers = getattr(config, "num_hash_layers", 0)
|
||||
self.is_hash = layer_id < n_hash_layers and not (is_deepseek_v4 and is_nextn)
|
||||
|
||||
@@ -957,7 +982,11 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids: Optional[torch.Tensor] = None,
|
||||
input_ids_global: Optional[torch.Tensor] = None,
|
||||
skip_shared_experts: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return_moe_output: bool = False,
|
||||
) -> Union[torch.Tensor, MoEOutput]:
|
||||
"""``return_moe_output`` asks for the pieces unmerged, as `MoEOutput`; only
|
||||
the TP paths honor it. Distinct from `get_forward().defer_moe_finalize`,
|
||||
the V2/V3 layer-boundary handoff."""
|
||||
from sglang.srt.layers.moe.mega_moe import forward_mega_moe, should_use_mega_moe
|
||||
|
||||
if should_use_mega_moe(self, hidden_states):
|
||||
@@ -998,6 +1027,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids_global=input_ids_global,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
use_vision_topk=use_vision_topk,
|
||||
return_moe_output=return_moe_output,
|
||||
)
|
||||
else:
|
||||
return self.forward_normal(
|
||||
@@ -1008,6 +1038,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
skip_shared_experts=skip_shared_experts,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
use_vision_topk=use_vision_topk,
|
||||
return_moe_output=return_moe_output,
|
||||
)
|
||||
else:
|
||||
return self.forward_deepep(
|
||||
@@ -1022,7 +1053,8 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids_global: Optional[torch.Tensor] = None,
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
use_vision_topk: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return_moe_output: bool = False,
|
||||
) -> Union[torch.Tensor, MoEOutput]:
|
||||
# Note(kpham-sgl): issue order satisfies 3 constraints:
|
||||
# - no stream explosion: main (routed) issued before alt block -> capture reuses 1 alt stream;
|
||||
# - PDL overlap: routed is the last main-stream kernel (fuses w/ residual add);
|
||||
@@ -1103,22 +1135,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
)
|
||||
ready = self.routed_quant_stream.record_event()
|
||||
routed_pre_quant_input = Mxfp8RoutedInputPreQuant(x_q, x_sf, ready)
|
||||
# The mHC post-split consumes the reduced row without an RMSNorm.
|
||||
use_fused_finalize_all_reduce = (
|
||||
self._fuse_finalize_all_reduce
|
||||
# The kernel completes the tensor-parallel sum itself.
|
||||
and self.reduce_results
|
||||
and has_shared_output
|
||||
and hidden_states.shape[-1] == 5120
|
||||
and not self._shared_expert_tp1
|
||||
and self.tp_size > 1
|
||||
and hidden_states.shape[0] <= _FUSED_FINALIZE_ALL_REDUCE_MAX_TOKENS
|
||||
and not should_skip_post_experts_all_reduce(is_tp_path=True)
|
||||
and should_use_fuse_finalize_all_reduce(
|
||||
self.experts, hidden_states.shape[0], hidden_states.shape[-1]
|
||||
)
|
||||
)
|
||||
deferred_finalize = use_fused_finalize_all_reduce or (
|
||||
deferred_finalize = (
|
||||
has_shared_output
|
||||
and not self._shared_expert_tp1
|
||||
and topk_output.format == TopKOutputFormat.BYPASSED
|
||||
@@ -1128,6 +1145,11 @@ class DeepseekV2MoE(nn.Module):
|
||||
or hidden_states.shape[0] <= self._deferred_finalize_max_tokens
|
||||
)
|
||||
)
|
||||
# NOTE: if `return_moe_output`, always try to defer finalize
|
||||
deferred_finalize = deferred_finalize or (
|
||||
return_moe_output
|
||||
and isinstance(self.experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod)
|
||||
)
|
||||
if deferred_finalize:
|
||||
final_hidden_states = self.experts.forward_deferred_finalize(
|
||||
hidden_states, topk_output, pre_quant_input=routed_pre_quant_input
|
||||
@@ -1174,91 +1196,24 @@ class DeepseekV2MoE(nn.Module):
|
||||
reduce=sum_post_experts_output,
|
||||
)
|
||||
|
||||
all_reduce_done = False
|
||||
if return_moe_output:
|
||||
return MoEOutput(
|
||||
routed=final_hidden_states,
|
||||
shared=shared_output,
|
||||
experts=self.experts,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
shared_is_replicated=self._shared_expert_tp1,
|
||||
)
|
||||
|
||||
if deferred_finalize:
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
finalize_flashinfer_trtllm_deferred_output,
|
||||
)
|
||||
|
||||
deferred = final_hidden_states
|
||||
if (
|
||||
use_fused_finalize_all_reduce
|
||||
and deferred.gemm2_out.shape[1] == hidden_states.shape[-1]
|
||||
):
|
||||
from sglang.kernels.ops.communication.all_reduce_fusion import (
|
||||
moe_finalize_all_reduce,
|
||||
)
|
||||
from sglang.srt.layers.moe.mhc_post_fusion import (
|
||||
current_mhc_post_fusion,
|
||||
)
|
||||
|
||||
mhc = current_mhc_post_fusion()
|
||||
if mhc is not None:
|
||||
from sglang.kernels.ops.communication.all_reduce_mhc import (
|
||||
moe_finalize_all_reduce_mhc,
|
||||
)
|
||||
|
||||
# Join the coefficients before the fused epilogue reads them.
|
||||
mhc.materialize_stats()
|
||||
if mhc.stats_stream is not None:
|
||||
current_stream.wait_stream(mhc.stats_stream)
|
||||
args = (
|
||||
deferred.gemm2_out,
|
||||
deferred.expanded_idx_to_permuted_idx,
|
||||
deferred.expert_weights,
|
||||
deferred.top_k,
|
||||
shared_output,
|
||||
mhc.residual,
|
||||
mhc.post,
|
||||
mhc.comb,
|
||||
)
|
||||
if mhc.combine_only:
|
||||
from sglang.kernels.ops.communication.all_reduce_mhc_combine import (
|
||||
moe_finalize_all_reduce_mhc_combine,
|
||||
)
|
||||
|
||||
final_hidden_states, mhc.output, mhc.combined = (
|
||||
moe_finalize_all_reduce_mhc_combine(
|
||||
*args, mhc.pre, world_size=self.tp_size
|
||||
)
|
||||
)
|
||||
elif mhc.norm_weight is not None:
|
||||
from sglang.kernels.ops.communication.all_reduce_mhc import (
|
||||
moe_finalize_all_reduce_mhc_quant,
|
||||
)
|
||||
|
||||
final_hidden_states, mhc.output, mhc.normalized, q, sf = (
|
||||
moe_finalize_all_reduce_mhc_quant(
|
||||
*args,
|
||||
mhc.pre,
|
||||
mhc.norm_weight,
|
||||
mhc.norm_eps,
|
||||
world_size=self.tp_size,
|
||||
)
|
||||
)
|
||||
mhc.quantized = (q, sf)
|
||||
else:
|
||||
final_hidden_states, mhc.output = moe_finalize_all_reduce_mhc(
|
||||
*args, world_size=self.tp_size
|
||||
)
|
||||
else:
|
||||
final_hidden_states = moe_finalize_all_reduce(
|
||||
deferred.gemm2_out,
|
||||
deferred.expanded_idx_to_permuted_idx,
|
||||
deferred.expert_weights,
|
||||
deferred.top_k,
|
||||
shared_output,
|
||||
world_size=self.tp_size,
|
||||
hidden_dim=hidden_states.shape[-1],
|
||||
# Routing metadata must be ready before it is consumed.
|
||||
prefetch_metadata=False,
|
||||
)
|
||||
all_reduce_done = True
|
||||
else:
|
||||
final_hidden_states = finalize_flashinfer_trtllm_deferred_output(
|
||||
deferred,
|
||||
shared_output,
|
||||
)
|
||||
final_hidden_states = finalize_flashinfer_trtllm_deferred_output(
|
||||
final_hidden_states,
|
||||
shared_output,
|
||||
)
|
||||
else:
|
||||
final_hidden_states = maybe_fuse_routed_scale_and_shared_add(
|
||||
self.experts,
|
||||
@@ -1266,24 +1221,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
None if self._shared_expert_tp1 else shared_output,
|
||||
self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
if not all_reduce_done and self.reduce_results:
|
||||
if (
|
||||
self.is_deepseek_v4
|
||||
and self.tp_size > 1
|
||||
and not should_skip_post_experts_all_reduce(is_tp_path=True)
|
||||
):
|
||||
from sglang.srt.layers.moe.mhc_post_fusion import (
|
||||
current_mhc_post_fusion,
|
||||
)
|
||||
|
||||
mhc = current_mhc_post_fusion()
|
||||
if _is_hip and _hip_moe.fused_all_reduce_mhc(
|
||||
self, mhc, final_hidden_states
|
||||
):
|
||||
return final_hidden_states
|
||||
if mhc is not None:
|
||||
mhc.start_stats_before_all_reduce()
|
||||
if self.reduce_results:
|
||||
final_hidden_states = post_experts_all_reduce(final_hidden_states)
|
||||
# TP1 shared experts are replicated: add them after this block's
|
||||
# all-reduce, or on TP rank 0 only when the stage boundary sums.
|
||||
@@ -1300,7 +1238,8 @@ class DeepseekV2MoE(nn.Module):
|
||||
skip_shared_experts: bool = False,
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
use_vision_topk: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return_moe_output: bool = False,
|
||||
) -> Union[torch.Tensor, MoEOutput]:
|
||||
if hasattr(self, "shared_experts") and use_intel_amx_backend(
|
||||
self.shared_experts.gate_up_proj
|
||||
):
|
||||
@@ -1435,30 +1374,22 @@ class DeepseekV2MoE(nn.Module):
|
||||
pre_quant_input=pre_quant_input,
|
||||
)
|
||||
|
||||
if return_moe_output:
|
||||
return MoEOutput(
|
||||
routed=final_hidden_states,
|
||||
shared=shared_output,
|
||||
experts=self.experts,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
shared_is_replicated=self._shared_expert_tp1,
|
||||
)
|
||||
|
||||
final_hidden_states = maybe_fuse_routed_scale_and_shared_add(
|
||||
self.experts,
|
||||
final_hidden_states,
|
||||
None if self._shared_expert_tp1 else shared_output,
|
||||
self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
if self.reduce_results:
|
||||
if (
|
||||
self.is_deepseek_v4
|
||||
and self.tp_size > 1
|
||||
and not should_skip_post_experts_all_reduce(is_tp_path=True)
|
||||
):
|
||||
from sglang.srt.layers.moe.mhc_post_fusion import (
|
||||
current_mhc_post_fusion,
|
||||
)
|
||||
|
||||
mhc = current_mhc_post_fusion()
|
||||
if _is_hip and _hip_moe.fused_all_reduce_mhc(
|
||||
self, mhc, final_hidden_states
|
||||
):
|
||||
return final_hidden_states
|
||||
if mhc is not None:
|
||||
mhc.start_stats_before_all_reduce()
|
||||
final_hidden_states = post_experts_all_reduce(final_hidden_states)
|
||||
# TP1 shared experts are replicated: add them after this block's
|
||||
# all-reduce, or on TP rank 0 only when the stage boundary sums.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -35,6 +35,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models import deepseek_v4_mhc as mhc
|
||||
from sglang.srt.models.dbrx import ReplicatedLinear
|
||||
from sglang.srt.models.deepseek_v4 import (
|
||||
DEEPSEEK_V4_STACKED_PARAMS_MAPPING,
|
||||
@@ -745,35 +746,31 @@ class DSparkV4Stage(DeepseekV4DecoderLayer):
|
||||
forward_batch: ForwardBatch,
|
||||
prev_pre: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
stats_stream = self._get_hc_stats_stream(hidden_states, forward_batch)
|
||||
# The draft stages run the plain four steps: no fused post, no seams, so
|
||||
# they need nothing from `bind_next` and never hand anything forward.
|
||||
assert self.attn_hc is not None and self.ffn_hc is not None
|
||||
stats_stream = None
|
||||
if mhc.use_stats_stream(self.hc_cfg, forward_batch, hidden_states):
|
||||
stats_stream = self.hc_stats_stream
|
||||
|
||||
residual = hidden_states
|
||||
x = self._hc_combine(
|
||||
hidden_states, prev_pre, self.input_layernorm, stats_stream
|
||||
)
|
||||
mhc.fork_stats_stream(stats_stream)
|
||||
x = mhc.combine(self.attn_hc, mhc.HcState(hidden_states, prev_pre))
|
||||
with self.self_attn.maybe_use_decode_attn_tp(forward_batch):
|
||||
x = self.self_attn(positions, x, forward_batch)
|
||||
attn_pre, attn_post, attn_comb = self._hc_mix_stats(
|
||||
hidden_states,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
self.hc_attn_base,
|
||||
stats_stream,
|
||||
attn_pre, attn_post, attn_comb = mhc.mix_stats(
|
||||
self.attn_hc, hidden_states, stats_stream
|
||||
)
|
||||
if stats_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(stats_stream)
|
||||
hidden_states = self.hc_post(x, residual, attn_post, attn_comb)
|
||||
|
||||
residual = hidden_states
|
||||
x = self._hc_combine(
|
||||
hidden_states, attn_pre, self.post_attention_layernorm, stats_stream
|
||||
)
|
||||
mhc.fork_stats_stream(stats_stream)
|
||||
x = mhc.combine(self.ffn_hc, mhc.HcState(hidden_states, attn_pre))
|
||||
x = self._run_ffn(x, forward_batch)
|
||||
ffn_pre, ffn_post, ffn_comb = self._hc_mix_stats(
|
||||
hidden_states,
|
||||
self.hc_ffn_fn,
|
||||
self.hc_ffn_scale,
|
||||
self.hc_ffn_base,
|
||||
stats_stream,
|
||||
ffn_pre, ffn_post, ffn_comb = mhc.mix_stats(
|
||||
self.ffn_hc, hidden_states, stats_stream
|
||||
)
|
||||
if stats_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(stats_stream)
|
||||
|
||||
@@ -0,0 +1,715 @@
|
||||
"""DeepSeek-V4.1 manifold-constrained hyper-connections (mHC).
|
||||
|
||||
The residual is ``R [T, hc_mult, hidden]`` -- hc_mult parallel streams per token --
|
||||
and each sublayer is wrapped in a hyper-connection (DeepSeek's ``attn_hc`` and
|
||||
``ffn_hc``) carrying a ``(pre, post, comb)`` triplet. Per hyper-connection:
|
||||
|
||||
1. mix stats predict ``(pre, post, comb)`` from ``R``
|
||||
2. combine collapse the streams with the PREVIOUS triplet's ``pre``, then norm
|
||||
3. sublayer attention or MoE, including its tensor-parallel reduction
|
||||
4. post ``R'[i] = post[i] * y + sum_j comb[j][i] * R[j]``
|
||||
|
||||
Step 1's ``pre`` is consumed by step 2 of the *next* hyper-connection -- the lag
|
||||
that lets step 1 run beside steps 2-3 and makes V4's fused post+pre inexpressible.
|
||||
``attn_hc -> ffn_hc`` stays inside a layer; ``ffn_hc -> attn_hc`` crosses layers,
|
||||
where an Engram or a row selection can replace ``R`` (`with_residual`/`take_rows`).
|
||||
|
||||
The layer owns parameters, parallelism and token-range policy; this module owns
|
||||
the kernel dispatch.
|
||||
|
||||
TODO: move dsv4 MHC into the same abstraction
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, NamedTuple, Optional, Tuple, TypeAlias, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.kernels.ops.layernorm.mhc_post_split_h import mhc_post_split_h
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.hardware_backend.npu.utils import is_npu_arch35
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.quantization.mxfp8_input import Mxfp8SwizzledInput
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.runner import compile_in_capture_mode
|
||||
from sglang.srt.models.deepseek_v2 import MoEOutput, _is_hip, _is_npu, _is_xpu
|
||||
from sglang.srt.runtime_context import get_parallel, get_platform
|
||||
from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
_is_gfx95_supported = is_gfx95_supported()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# What a layer hands this module
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
HcTriplet = Tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
|
||||
|
||||
class HcConfig(NamedTuple):
|
||||
"""Everything about a layer's mHC that is fixed for the model's lifetime."""
|
||||
|
||||
mult: int
|
||||
sinkhorn_iters: int
|
||||
eps: float
|
||||
rms_eps: float
|
||||
hidden: int
|
||||
pre_from_prev: bool
|
||||
cp_prefill: bool
|
||||
|
||||
|
||||
class HcSubLayer(NamedTuple):
|
||||
"""One hyper-connection's fixed material. Holds the layer's Parameters by
|
||||
reference; a plain tuple registers nothing, so checkpoint keys stay put."""
|
||||
|
||||
cfg: HcConfig
|
||||
fn: torch.Tensor
|
||||
scale: torch.Tensor
|
||||
base: torch.Tensor
|
||||
norm: RMSNorm
|
||||
tf32_parts: Optional[Any] = None
|
||||
bf16_parts: Optional[Any] = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# How far the previous hyper-connection got toward the next sublayer's input
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class HcNormed(NamedTuple):
|
||||
"""Summed and normed; the sublayer can take these rows as they are."""
|
||||
|
||||
rows: torch.Tensor
|
||||
|
||||
|
||||
class HcQuantized(NamedTuple):
|
||||
"""Normed, and already swizzled for the sublayer's first projection."""
|
||||
|
||||
rows: torch.Tensor
|
||||
swizzled: Mxfp8SwizzledInput
|
||||
|
||||
|
||||
HcPreOutput: TypeAlias = Union[HcNormed, HcQuantized]
|
||||
"""The next sublayer's input, built by the previous post; a receiver skips step 2
|
||||
(and ``pre``) entirely. Always normed -- combine and norm travel together."""
|
||||
|
||||
|
||||
class AttnOutput(NamedTuple):
|
||||
"""Attention's wo_b rows with the TP reduction left to the fused post."""
|
||||
|
||||
partial: torch.Tensor
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The real parameters: what crosses from one hyper-connection to the next
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class HcPending(NamedTuple):
|
||||
"""A post not run yet; the residual it would produce is implied. The backward
|
||||
counterpart of `HcPreOutput`: the HIP boundary kernel takes both and runs the
|
||||
previous post, this triplet and this collapse in one launch."""
|
||||
|
||||
rows: torch.Tensor
|
||||
residual: torch.Tensor
|
||||
post: torch.Tensor
|
||||
comb: torch.Tensor
|
||||
|
||||
|
||||
class HcState(msgspec.Struct):
|
||||
"""The mHC state of the stream between two hyper-connections.
|
||||
|
||||
``streams`` is the residual ``R``, or -- out of the HIP fused boundary -- the
|
||||
post that would rebuild it, still outstanding. Mutable for one reason:
|
||||
`release` must empty a consumed state in every frame that still binds it.
|
||||
"""
|
||||
|
||||
streams: Union[torch.Tensor, HcPending, None]
|
||||
pre: Optional[torch.Tensor] = None
|
||||
input: Optional[HcPreOutput] = None
|
||||
|
||||
def release(self) -> None:
|
||||
"""Empty the state at its last reader, the combine; the residual the post
|
||||
still needs lives on through the caller's alias. Stale holders -- above
|
||||
all the model loop's binding across the layer call -- then stop pinning
|
||||
the dead residual through the next sublayer."""
|
||||
self.streams = None
|
||||
self.pre = None
|
||||
self.input = None
|
||||
|
||||
@property
|
||||
def residual(self) -> torch.Tensor:
|
||||
"""The materialized streams; run an outstanding post first (`materialized`)."""
|
||||
assert isinstance(self.streams, torch.Tensor)
|
||||
return self.streams
|
||||
|
||||
def materialized(self, cfg: HcConfig) -> HcState:
|
||||
"""Run the outstanding post, if any; residual readers call this first."""
|
||||
if not isinstance(self.streams, HcPending):
|
||||
return self
|
||||
p = self.streams
|
||||
return HcState(post(cfg, p.rows, p.residual, p.post, p.comb), self.pre)
|
||||
|
||||
def with_residual(self, residual: torch.Tensor) -> HcState:
|
||||
"""The residual was rewritten: drop ``input``; ``pre`` weights streams, not
|
||||
rows, so it survives."""
|
||||
return HcState(residual, self.pre)
|
||||
|
||||
def take_rows(self, rows: Callable[[torch.Tensor], torch.Tensor]) -> HcState:
|
||||
"""Keep only the rows ``rows`` selects -- the late-layer tail's narrowing."""
|
||||
pre = None if self.pre is None else rows(self.pre)
|
||||
assert self.streams is not None
|
||||
if isinstance(self.streams, HcPending):
|
||||
return HcState(HcPending(*(rows(t) for t in self.streams)), pre)
|
||||
else:
|
||||
return HcState(rows(self.streams), pre)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The plan: how a hyper-connection intends to run its post
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class HcNextBoundary(NamedTuple):
|
||||
"""What the next hyper-connection can take off this one's post; resolved after
|
||||
weights load. None where the seam is closed (last layer, Engram)."""
|
||||
|
||||
norm: RMSNorm
|
||||
accepts_mxfp8: bool
|
||||
norm_fusable: bool
|
||||
|
||||
|
||||
def is_deferred_finalize(routed) -> bool:
|
||||
"""Whether a `MoEOutput` really carries an unfinalized handle for the post."""
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
FlashInferTrtllmDeferredFinalizeOutput,
|
||||
)
|
||||
|
||||
return isinstance(routed, FlashInferTrtllmDeferredFinalizeOutput)
|
||||
|
||||
|
||||
def make_boundary(norm: RMSNorm, *, accepts_mxfp8: bool) -> HcNextBoundary:
|
||||
"""Can this hyper-connection's norm fold into the previous post, and does its
|
||||
sublayer take a pre-quantized input."""
|
||||
return HcNextBoundary(
|
||||
norm=norm,
|
||||
accepts_mxfp8=accepts_mxfp8,
|
||||
norm_fusable=(
|
||||
not norm.cast_x_before_out_mul
|
||||
and norm.variance_size_override is None
|
||||
and norm.weight.dtype == torch.bfloat16
|
||||
# TODO: support other norm shape
|
||||
and norm.weight.shape == (5120,)
|
||||
and norm.weight.is_contiguous()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 1 -- the mix GEMM and the triplet it splits into
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def use_stats_stream(cfg: HcConfig, forward_batch: ForwardBatch, x: torch.Tensor):
|
||||
if not cfg.pre_from_prev:
|
||||
return False
|
||||
decode_like = forward_batch.forward_mode.is_decode() or (
|
||||
forward_batch.forward_mode.is_target_verify() and x.shape[0] > 0
|
||||
)
|
||||
return decode_like and (
|
||||
not get_platform().is_sm90
|
||||
or x.shape[0] == 1
|
||||
or (forward_batch.forward_mode.is_decode() and 1 < x.shape[0] <= 64)
|
||||
)
|
||||
|
||||
|
||||
def mix_stats(
|
||||
hc: HcSubLayer, x: torch.Tensor, stats_stream: Optional[torch.cuda.Stream] = None
|
||||
) -> HcTriplet:
|
||||
"""Predict the triplet on the caller's stream or the stats stream."""
|
||||
if stats_stream is None:
|
||||
return _mix_stats_impl(hc, x)
|
||||
main_stream = torch.cuda.current_stream()
|
||||
x.record_stream(stats_stream)
|
||||
with torch.cuda.stream(stats_stream):
|
||||
coefficients = _mix_stats_impl(hc, x)
|
||||
for coefficient in coefficients:
|
||||
coefficient.record_stream(main_stream)
|
||||
return coefficients
|
||||
|
||||
|
||||
def _mix_stats_impl(hc: HcSubLayer, x: torch.Tensor) -> HcTriplet:
|
||||
from sglang.kernels.ops.layernorm.mhc import (
|
||||
hc_mix_stats,
|
||||
hc_mix_stats_sinkhorn,
|
||||
hc_split_sinkhorn,
|
||||
)
|
||||
|
||||
cfg = hc.cfg
|
||||
|
||||
x_flat = x.flatten(1)
|
||||
|
||||
from sglang.srt.batch_invariant_ops import (
|
||||
is_batch_invariant_mode_enabled,
|
||||
)
|
||||
|
||||
parts = bf16_parts = None
|
||||
hopper_medium = get_platform().is_sm90 and 32 <= x_flat.shape[0] < 4096
|
||||
if (
|
||||
x.is_cuda
|
||||
and (x_flat.shape[0] >= 128 or hopper_medium)
|
||||
and x_flat.is_contiguous()
|
||||
and (get_platform().is_sm100 or get_platform().is_sm90)
|
||||
and envs.SGLANG_OPT_DEEPGEMM_HC_PRENORM.get()
|
||||
and not is_batch_invariant_mode_enabled()
|
||||
):
|
||||
parts, bf16_parts = hc.tf32_parts, hc.bf16_parts
|
||||
|
||||
use_bf16_projection = bf16_parts is not None and (
|
||||
hopper_medium or 4096 <= x_flat.shape[0] <= 65536
|
||||
)
|
||||
hopper_fused_stats = get_platform().is_sm90 and (
|
||||
x.shape[0] == 1 or (bf16_parts is not None and 32 <= x.shape[0] <= 65536)
|
||||
)
|
||||
# gfx950 uses the fused Triton port at every row count (MI350X).
|
||||
use_fused_stats = _is_gfx95_supported or (
|
||||
torch.version.cuda is not None
|
||||
and (get_platform().is_blackwell or hopper_fused_stats)
|
||||
)
|
||||
if x.is_cuda and use_fused_stats and x.dtype == torch.bfloat16:
|
||||
# The default split-K/Sinkhorn fusion preserves batch invariance;
|
||||
# compensated projections above are disabled in batch-invariant mode.
|
||||
if use_bf16_projection:
|
||||
from sglang.kernels.ops.layernorm.mhc import (
|
||||
hc_mix_stats_sinkhorn_bf16x3,
|
||||
)
|
||||
|
||||
pre, post, comb = hc_mix_stats_sinkhorn_bf16x3(
|
||||
x_flat,
|
||||
bf16_parts,
|
||||
hc.scale,
|
||||
hc.base,
|
||||
cfg.sinkhorn_iters,
|
||||
cfg.rms_eps,
|
||||
cfg.eps,
|
||||
)
|
||||
elif parts is not None:
|
||||
from sglang.kernels.ops.layernorm.mhc import (
|
||||
hc_mix_stats_sinkhorn_deepgemm,
|
||||
)
|
||||
|
||||
pre, post, comb = hc_mix_stats_sinkhorn_deepgemm(
|
||||
x_flat,
|
||||
parts,
|
||||
hc.scale,
|
||||
hc.base,
|
||||
cfg.sinkhorn_iters,
|
||||
cfg.rms_eps,
|
||||
cfg.eps,
|
||||
)
|
||||
else:
|
||||
pre, post, comb = hc_mix_stats_sinkhorn(
|
||||
x_flat,
|
||||
hc.fn,
|
||||
hc.scale,
|
||||
hc.base,
|
||||
cfg.mult,
|
||||
cfg.sinkhorn_iters,
|
||||
cfg.rms_eps,
|
||||
cfg.eps,
|
||||
)
|
||||
return pre, post, comb
|
||||
if x.is_cuda and torch.version.cuda is not None:
|
||||
# cuBLAS/torch reductions can change order with num_tokens; this kernel
|
||||
# keeps the mixing and RMS reductions batch-invariant.
|
||||
mixes = hc_mix_stats(x_flat, hc.fn, cfg.rms_eps).unsqueeze(1)
|
||||
else:
|
||||
x_flat = x_flat.float()
|
||||
rsqrt = torch.rsqrt(x_flat.square().mean(-1, keepdim=True) + cfg.rms_eps)
|
||||
mixes = (F.linear(x_flat, hc.fn) * rsqrt).unsqueeze(1)
|
||||
if _is_xpu:
|
||||
from sglang.srt.models.deepseek_v4 import get_mhc_ops
|
||||
|
||||
split_sinkhorn = get_mhc_ops().hc_split_sinkhorn
|
||||
else:
|
||||
split_sinkhorn = hc_split_sinkhorn
|
||||
pre, post, comb = split_sinkhorn(
|
||||
mixes,
|
||||
hc.scale,
|
||||
hc.base,
|
||||
cfg.mult,
|
||||
cfg.sinkhorn_iters,
|
||||
cfg.eps,
|
||||
)
|
||||
return pre.squeeze(1), post.squeeze(1), comb.squeeze(1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 2 -- collapse the streams and normalize
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def fork_stats_stream(stats_stream: Optional[torch.cuda.Stream]) -> None:
|
||||
"""Let the side stream pick up from here; the layers call it right before
|
||||
each combine (fork position measured indistinguishable across batch sizes)."""
|
||||
if stats_stream is not None:
|
||||
stats_stream.wait_stream(torch.cuda.current_stream())
|
||||
|
||||
|
||||
def combine(
|
||||
hc: HcSubLayer,
|
||||
state: HcState,
|
||||
quantized: Optional[list] = None,
|
||||
) -> torch.Tensor:
|
||||
"""The sublayer's input: ``norm(sum_k pre[k] * R[k])``, short-circuited by
|
||||
whatever the previous post left in ``state.input``."""
|
||||
shortcut = state.input
|
||||
if isinstance(shortcut, HcQuantized):
|
||||
assert quantized is not None
|
||||
quantized.append(shortcut.swizzled)
|
||||
return shortcut.rows
|
||||
if isinstance(shortcut, HcNormed):
|
||||
return shortcut.rows
|
||||
return _combine(hc, state, quantized)
|
||||
|
||||
|
||||
def _combine(
|
||||
hc: HcSubLayer,
|
||||
state: HcState,
|
||||
quantized: Optional[list],
|
||||
) -> torch.Tensor:
|
||||
"""The combine computed from the residual's rows."""
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_combine
|
||||
|
||||
cfg, norm = hc.cfg, hc.norm
|
||||
x, apply_pre = state.residual, state.pre
|
||||
quantize = quantized is not None
|
||||
x_flat = x.flatten(1)
|
||||
|
||||
if apply_pre is None:
|
||||
return norm(x[:, 0, :])
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
|
||||
hopper_fused = (
|
||||
get_platform().is_sm90
|
||||
and not quantize
|
||||
and (0 < x.shape[0] <= 96 or 4096 <= x.shape[0] <= 65536)
|
||||
and not is_batch_invariant_mode_enabled()
|
||||
)
|
||||
if (
|
||||
x.is_cuda
|
||||
and (get_platform().is_blackwell or hopper_fused)
|
||||
and x.dtype == torch.bfloat16
|
||||
and apply_pre.stride(1) == 1
|
||||
and cfg.mult == 4
|
||||
and cfg.hidden == 5120
|
||||
and norm.weight.dtype == torch.bfloat16
|
||||
and not norm.cast_x_before_out_mul
|
||||
and norm.variance_size_override is None
|
||||
):
|
||||
# One fused form for every row count. Each row's reduction order is
|
||||
# fixed regardless of the grid split, so this holds under
|
||||
# batch-invariant mode too.
|
||||
if quantize and x.shape[0] <= 128:
|
||||
from sglang.kernels.ops.layernorm.hc_combine_norm import (
|
||||
hc_combine_norm_mxfp8,
|
||||
)
|
||||
|
||||
y, y_q, y_sf = hc_combine_norm_mxfp8(
|
||||
x_flat, apply_pre, norm.weight, norm.variance_epsilon
|
||||
)
|
||||
quantized.append(Mxfp8SwizzledInput(y_q, y_sf))
|
||||
return y
|
||||
from sglang.kernels.ops.layernorm.hc_combine_norm import hc_combine_norm
|
||||
|
||||
return hc_combine_norm(x_flat, apply_pre, norm.weight, norm.variance_epsilon)
|
||||
return norm(hc_combine(x_flat, apply_pre, cfg.mult, x.dtype))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 4 -- rebuild the streams, optionally collapsing the next input too
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def post(
|
||||
cfg: HcConfig,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_mix: torch.Tensor,
|
||||
comb: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""``R'[i] = post[i] * x + sum_j comb[j][i] * R[j]``, by whichever kernel fits."""
|
||||
if x.shape[0] == 0:
|
||||
return torch.empty((0, cfg.mult, x.shape[-1]), dtype=x.dtype, device=x.device)
|
||||
|
||||
if _is_npu:
|
||||
if not is_npu_arch35():
|
||||
return torch.ops.custom.npu_hc_post(x, residual, post_mix, comb)
|
||||
# The A5 build of npu_hc_post is batched -- it requires a leading batch axis
|
||||
# on every operand.
|
||||
return torch.ops.custom.npu_hc_post(
|
||||
x.unsqueeze(0),
|
||||
residual.unsqueeze(0),
|
||||
post_mix.unsqueeze(0),
|
||||
comb.unsqueeze(0),
|
||||
).squeeze(0)
|
||||
|
||||
if _is_xpu:
|
||||
from sglang.srt.models.deepseek_v4 import get_mhc_ops
|
||||
|
||||
return get_mhc_ops().mhc_post(x, residual, post_mix, comb)
|
||||
|
||||
if (
|
||||
get_platform().is_blackwell
|
||||
and cfg.pre_from_prev
|
||||
and cfg.mult == 4
|
||||
and cfg.hidden == 5120
|
||||
# This 384 is the kernel's own tile limit, unrelated to the collective's
|
||||
# push slots; it covers every narrow batch, not one particular regime.
|
||||
and x.shape[0] <= 384
|
||||
):
|
||||
return mhc_post_split_h(x, residual, post_mix, comb)
|
||||
|
||||
if _is_hip:
|
||||
from sglang.srt.models.deepseek_common.amd import deepseek_v4_hip as _hip
|
||||
|
||||
y = _hip.hc_post(cfg, x, residual, post_mix, comb)
|
||||
if y is not None:
|
||||
return y
|
||||
|
||||
if envs.SGLANG_OPT_USE_FLASHINFER_MHC.get():
|
||||
from flashinfer.mhc import mhc_post
|
||||
|
||||
return mhc_post(x, residual, post_mix, comb)
|
||||
|
||||
if envs.SGLANG_OPT_USE_TILELANG_MHC_POST.get():
|
||||
if (
|
||||
cfg.pre_from_prev
|
||||
and get_platform().is_sm90
|
||||
and cfg.mult == 4
|
||||
and cfg.hidden == 5120
|
||||
and 1 <= x.shape[0] <= 64
|
||||
):
|
||||
return mhc_post_split_h(x, residual, post_mix, comb)
|
||||
|
||||
from sglang.kernels.ops.layernorm.mhc import mhc_post
|
||||
|
||||
return mhc_post(x, residual, post_mix, comb)
|
||||
|
||||
if _is_hip:
|
||||
from aiter.ops.mhc import mhc_post
|
||||
|
||||
result = torch.empty_like(residual)
|
||||
mhc_post(result, x, residual, post_mix, comb)
|
||||
return result
|
||||
|
||||
assert residual.shape == (x.shape[0], cfg.mult, x.shape[-1])
|
||||
assert post_mix.shape == (x.shape[0], cfg.mult)
|
||||
assert comb.shape == (x.shape[0], cfg.mult, cfg.mult)
|
||||
|
||||
@compile_in_capture_mode
|
||||
def post_torch_impl(x, residual, post_mix, comb):
|
||||
return (
|
||||
post_mix.unsqueeze(-1) * x.unsqueeze(1)
|
||||
+ (comb.unsqueeze(-1) * residual.unsqueeze(2)).sum(dim=1)
|
||||
).type_as(x)
|
||||
|
||||
return post_torch_impl(x, residual, post_mix, comb)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Planning the post
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def can_fuse_post(cfg: HcConfig) -> bool:
|
||||
"""Whether a fused post is expressible at all for this model, device and
|
||||
parallel layout.
|
||||
|
||||
The capability half of the decision; which token ranges should take it, and what
|
||||
batch-invariant mode should do about it, are the caller's policy.
|
||||
"""
|
||||
return (
|
||||
get_platform().is_blackwell
|
||||
and cfg.pre_from_prev
|
||||
and cfg.mult == 4
|
||||
and cfg.hidden == 5120
|
||||
and get_parallel().attn_dp_size == 1
|
||||
and not cfg.cp_prefill
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Running the post
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compute_triplet(
|
||||
hc: HcSubLayer,
|
||||
residual: torch.Tensor,
|
||||
stats_stream: Optional[torch.cuda.Stream],
|
||||
) -> HcTriplet:
|
||||
"""Issue the triplet (on the side stream, beside the sublayer) and join before
|
||||
the post reads it."""
|
||||
coefficients = mix_stats(hc, residual, stats_stream)
|
||||
if stats_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(stats_stream)
|
||||
return coefficients
|
||||
|
||||
|
||||
def _post_fusion(
|
||||
hc: HcSubLayer,
|
||||
y: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
coefficients: HcTriplet,
|
||||
next: Optional[HcNextBoundary],
|
||||
) -> HcState:
|
||||
"""Step 4 without a collective: the wide-tile kernel folds the next combine +
|
||||
norm in where it serves this seam, otherwise the pure post runs and the next
|
||||
combine computes itself."""
|
||||
cfg = hc.cfg
|
||||
pre, post_mix, comb = coefficients
|
||||
if (
|
||||
next is not None
|
||||
and next.norm_fusable
|
||||
and cfg.pre_from_prev
|
||||
and get_platform().is_blackwell
|
||||
and 4096 <= y.shape[0] <= 65536
|
||||
and cfg.hidden == 5120
|
||||
and cfg.mult == 4
|
||||
and get_parallel().attn_dp_size == 1
|
||||
and not cfg.cp_prefill
|
||||
):
|
||||
from sglang.kernels.ops.layernorm.mhc_post_combine_norm_prefill import (
|
||||
mhc_post_combine_norm_prefill,
|
||||
)
|
||||
|
||||
updated, normalized = mhc_post_combine_norm_prefill(
|
||||
y,
|
||||
residual,
|
||||
post_mix,
|
||||
comb,
|
||||
pre,
|
||||
next.norm.weight,
|
||||
next.norm.variance_epsilon,
|
||||
)
|
||||
return HcState(updated, pre, HcNormed(normalized))
|
||||
return HcState(post(cfg, y, residual, post_mix, comb), pre)
|
||||
|
||||
|
||||
def run_attn_post(
|
||||
hc: HcSubLayer,
|
||||
out: Union[torch.Tensor, AttnOutput],
|
||||
residual: torch.Tensor,
|
||||
*,
|
||||
stats_stream: Optional[torch.cuda.Stream],
|
||||
next: Optional[HcNextBoundary],
|
||||
world_size: int,
|
||||
) -> HcState:
|
||||
"""The attention post. An `AttnOutput` rides the collective kernel (which also
|
||||
folds ``next``'s norm); the attention may decline the handover even when asked,
|
||||
so the type is the ground truth."""
|
||||
if isinstance(out, AttnOutput):
|
||||
from sglang.kernels.ops.communication.all_reduce_mhc import (
|
||||
all_reduce_mhc_post_combine_norm,
|
||||
)
|
||||
|
||||
assert next is not None
|
||||
pre, post_mix, comb = _compute_triplet(hc, residual, stats_stream)
|
||||
_, updated, normalized = all_reduce_mhc_post_combine_norm(
|
||||
out.partial,
|
||||
residual,
|
||||
post_mix,
|
||||
comb,
|
||||
pre,
|
||||
next.norm.weight,
|
||||
next.norm.variance_epsilon,
|
||||
world_size=world_size,
|
||||
)
|
||||
return HcState(updated, pre, HcNormed(normalized))
|
||||
coefficients = _compute_triplet(hc, residual, stats_stream)
|
||||
return _post_fusion(hc, out, residual, coefficients, next)
|
||||
|
||||
|
||||
def run_moe_post(
|
||||
hc: HcSubLayer,
|
||||
out: Union[torch.Tensor, MoEOutput],
|
||||
residual: torch.Tensor,
|
||||
*,
|
||||
stats_stream: Optional[torch.cuda.Stream],
|
||||
next: Optional[HcNextBoundary],
|
||||
world_size: int,
|
||||
) -> HcState:
|
||||
"""The MoE post. A deferred finalize the push plane can carry rides the
|
||||
collective kernel (finalize + shared add + all-reduce, quantizing ``next``'s
|
||||
input when a boundary is given); anything else is finalized here and takes
|
||||
the plain post."""
|
||||
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
|
||||
can_fuse_all_reduce,
|
||||
)
|
||||
|
||||
if (
|
||||
isinstance(out, MoEOutput)
|
||||
and is_deferred_finalize(out.routed)
|
||||
# expert_weights rows are the true token count; gemm2_out is the expanded
|
||||
# [T x top_k (padded)] view and overstates the plane load by ~6x.
|
||||
and can_fuse_all_reduce(out.routed.expert_weights.shape[0], hc.cfg.hidden)
|
||||
):
|
||||
pre, post_mix, comb = _compute_triplet(hc, residual, stats_stream)
|
||||
args = (
|
||||
out.routed.gemm2_out,
|
||||
out.routed.expanded_idx_to_permuted_idx,
|
||||
out.routed.expert_weights,
|
||||
out.routed.top_k,
|
||||
out.shared,
|
||||
residual,
|
||||
post_mix,
|
||||
comb,
|
||||
)
|
||||
if (
|
||||
next is not None
|
||||
and next.accepts_mxfp8
|
||||
and next.norm_fusable
|
||||
# The quant epilogue supports at most 128 batch size
|
||||
and out.routed.expert_weights.shape[0] <= 128
|
||||
):
|
||||
from sglang.kernels.ops.communication.all_reduce_mhc import (
|
||||
moe_finalize_all_reduce_mhc_post_combine_norm_quant,
|
||||
)
|
||||
|
||||
_, updated, normalized, q, sf = (
|
||||
moe_finalize_all_reduce_mhc_post_combine_norm_quant(
|
||||
*args,
|
||||
pre,
|
||||
next.norm.weight,
|
||||
next.norm.variance_epsilon,
|
||||
world_size=world_size,
|
||||
)
|
||||
)
|
||||
return HcState(
|
||||
updated, pre, HcQuantized(normalized, Mxfp8SwizzledInput(q, sf))
|
||||
)
|
||||
from sglang.kernels.ops.communication.all_reduce_mhc import (
|
||||
moe_finalize_all_reduce_mhc_post,
|
||||
)
|
||||
|
||||
_, updated = moe_finalize_all_reduce_mhc_post(*args, world_size=world_size)
|
||||
return HcState(updated, pre)
|
||||
if isinstance(out, MoEOutput):
|
||||
from sglang.srt.layers.moe import post_experts_all_reduce
|
||||
from sglang.srt.layers.moe.utils import should_add_replicated_moe_output
|
||||
|
||||
# The post boundary owns the sum; a replicated shared expert is added
|
||||
# after the reduction so it is counted only once across TP ranks.
|
||||
pieces = out
|
||||
out = post_experts_all_reduce(pieces.get_merged())
|
||||
if (
|
||||
pieces.shared is not None
|
||||
and pieces.shared_is_replicated
|
||||
and should_add_replicated_moe_output()
|
||||
):
|
||||
out += pieces.shared
|
||||
coefficients = _compute_triplet(hc, residual, stats_stream)
|
||||
return _post_fusion(hc, out, residual, coefficients, next)
|
||||
Reference in New Issue
Block a user