[Performance] Optimize Qwen3.8-Flash-Next BF16 decode on Blackwell (#42175)

This commit is contained in:
Xiaoyu Zhang
2026-10-09 18:46:12 +08:00
committed by GitHub
parent 516fd1c1b5
commit 0eb24e7d5b
21 changed files with 943 additions and 73 deletions
@@ -68,13 +68,14 @@ SGL_DEVICE void barrier_cluster_wait() {
template <uint32_t kWorldSize, typename WeightT>
struct MoeFinalizeAllReduceParams {
bf16_t* out; // [num_tokens, kHiddenDim], output-only
const bf16_t* gemm2; // [P, kHiddenDim], permuted / padded rows
const int32_t* idx; // [num_tokens * kTopK], -1 = dropped slot
const WeightT* weights; // [num_tokens, kTopK], scaling already folded in
const bf16_t* shared; // [num_tokens, kHiddenDim] (kHasShared only)
const bf16_t* norm_weight; // [kHiddenDim] (kNorm only)
float norm_eps; // kNorm only
bf16_t* out; // [num_tokens, kHiddenDim], output-only
const bf16_t* gemm2; // [P, kHiddenDim], permuted / padded rows
const int32_t* idx; // [num_tokens * kTopK], -1 = dropped slot
const WeightT* weights; // [num_tokens, kTopK], scaling already folded in
const bf16_t* shared; // [num_tokens, kHiddenDim] (kHasShared only)
const float* shared_gate = nullptr; // Optional FP32 sigmoid, one per token.
const bf16_t* norm_weight; // [kHiddenDim] (kNorm only)
float norm_eps; // kNorm only
// Caller's promise that everything read before the PDL wait is complete when
// the predecessor merely *triggers*: no all-reduce on this plane right before
// it, and the routing metadata's producers finished (PDL completion is not
@@ -199,7 +200,14 @@ struct RowClusterTrait {
// --- stage 1: the deferred finalize of one 16B vector ------------------------
// The shared-expert vector is loaded first so that load is in flight while the
// routing rows and the kTopK gathers are fetched.
template <uint32_t kHiddenDim, uint32_t kTopK, bool kHasShared, bool kUsePDL, uint32_t kWorldSize, typename WeightT>
template <
uint32_t kHiddenDim,
uint32_t kTopK,
bool kHasShared,
bool kUsePDL,
uint32_t kWorldSize,
typename WeightT,
bool kGatedShared = false>
SGL_DEVICE StageVec
finalize_vec(const MoeFinalizeAllReduceParams<kWorldSize, WeightT>& params, uint32_t token, uint32_t hvec) {
using namespace device;
@@ -263,7 +271,12 @@ finalize_vec(const MoeFinalizeAllReduceParams<kWorldSize, WeightT>& params, uint
if constexpr (kHasShared) {
const auto routed = cast<fp32x2_t>(cast<bf16x2_t>(acc[j]));
const auto sh = cast<fp32x2_t>(shared_in[j]);
out[j] = cast<bf16x2_t>(fp32x2_t{routed.x + sh.x, routed.y + sh.y});
if constexpr (kGatedShared) {
const float gate = params.shared_gate[token];
out[j] = cast<bf16x2_t>(fp32x2_t{fmaf(gate, sh.x, routed.x), fmaf(gate, sh.y, routed.y)});
} else {
out[j] = cast<bf16x2_t>(fp32x2_t{routed.x + sh.x, routed.y + sh.y});
}
} else {
out[j] = cast<bf16x2_t>(acc[j]);
}
@@ -287,7 +300,8 @@ template <
typename WeightT,
bool kMhc = false,
bool kQuant = false,
bool kCollapse = false>
bool kCollapse = false,
bool kGatedShared = false>
__global__ __launch_bounds__(RowClusterTrait<kHiddenDim, kClusterSize>::kBlockSize)
__cluster_dims__(1, kClusterSize, 1) void moe_finalize_all_reduce_kernel(
const __grid_constant__ MoeFinalizeAllReduceParams<kWorldSize, WeightT> params) {
@@ -342,7 +356,8 @@ __global__ __launch_bounds__(RowClusterTrait<kHiddenDim, kClusterSize>::kBlockSi
// stage 1: finalize this row's vector in registers and push it to every peer
const auto vid = row_idx * kRowVecs + hvec;
{
auto vec = finalize_vec<kHiddenDim, kTopK, kHasShared, kUsePDL>(params, row_idx, hvec);
auto vec =
finalize_vec<kHiddenDim, kTopK, kHasShared, kUsePDL, kWorldSize, WeightT, kGatedShared>(params, row_idx, hvec);
Lamport::clear_pos_zero(vec.data());
#pragma unroll
for (uint32_t i = 0; i < kWorldSize; ++i) {
@@ -456,7 +471,8 @@ template <
typename WeightT,
bool kMhc = false,
bool kQuant = false,
bool kCollapse = false>
bool kCollapse = false,
bool kGatedShared = false>
struct MoeFinalizeAllReduceKernel {
private:
static_assert(std::is_same_v<WeightT, bf16_t> || std::is_same_v<WeightT, fp32_t>);
@@ -476,7 +492,8 @@ struct MoeFinalizeAllReduceKernel {
WeightT,
kMhc,
kQuant,
kCollapse>;
kCollapse,
kGatedShared>;
public:
/// out = [allreduce over ranks of] finalize(gemm2_out, idx, weights) [+ shared] [-> RMSNorm(norm_weight, eps)].
@@ -509,6 +526,37 @@ struct MoeFinalizeAllReduceKernel {
std::nullopt);
}
// Preserve routed BF16 rounding, then apply the FP32 shared gate with FMA.
static void run_shared_gate(
CommunicatorRef ref,
TensorView out,
TensorView gemm2_out,
TensorView permuted_idx,
TensorView expert_weights,
TensorView shared_output,
TensorView shared_gate) {
static_assert(kGatedShared && !kMhc);
run_impl(
ref,
out,
gemm2_out,
permuted_idx,
expert_weights,
shared_output,
std::nullopt,
0.0,
false,
std::nullopt,
std::nullopt,
std::nullopt,
std::nullopt,
std::nullopt,
std::nullopt,
std::nullopt,
std::nullopt,
shared_gate);
}
/// Finalize + all-reduce + HC=4 post; original reduced output is retained.
static void run_mhc_post(
CommunicatorRef ref,
@@ -628,7 +676,8 @@ struct MoeFinalizeAllReduceKernel {
std::optional<TensorView> pre = std::nullopt,
std::optional<TensorView> normalized = std::nullopt,
std::optional<TensorView> quantized = std::nullopt,
std::optional<TensorView> scales = std::nullopt) {
std::optional<TensorView> scales = std::nullopt,
std::optional<TensorView> shared_gate = std::nullopt) {
using namespace host;
const auto& comm = *ref.get();
const auto& push = comm.get_push_obj();
@@ -664,6 +713,10 @@ struct MoeFinalizeAllReduceKernel {
.with_device<kDLCUDA>(device)
.verify(shared_output.value());
}
if constexpr (kGatedShared) {
CHECK_HOST(shared_output.has_value() && shared_gate.has_value());
TensorMatcher({T}).with_dtype<fp32_t>().with_device<kDLCUDA>(device).verify(shared_gate.value());
}
if (norm_weight.has_value()) {
TensorMatcher({kHiddenDim})
.with_strides({1})
@@ -717,6 +770,7 @@ struct MoeFinalizeAllReduceKernel {
.idx = static_cast<const int32_t*>(permuted_idx.data_ptr()),
.weights = static_cast<const WeightT*>(expert_weights.data_ptr()),
.shared = shared_output.has_value() ? static_cast<const bf16_t*>(shared_output.value().data_ptr()) : nullptr,
.shared_gate = shared_gate.has_value() ? static_cast<const float*>(shared_gate.value().data_ptr()) : nullptr,
.norm_weight = norm_weight.has_value() ? static_cast<const bf16_t*>(norm_weight.value().data_ptr()) : nullptr,
.norm_eps = static_cast<float>(eps),
.prefetch_metadata = prefetch_metadata,
@@ -0,0 +1,159 @@
#pragma once
#include "hc_combine.cuh"
namespace sglang {
// Expose the existing two stock stages separately; all arithmetic is unchanged.
struct HcCombineDecode {
static void gate(tvm::ffi::TensorView normed, tvm::ffi::TensorView weight, tvm::ffi::TensorView partials) {
using namespace host;
SymbolicSize m{"rows"};
SymbolicDevice dev;
dev.set_options<kDLCUDA>();
TensorMatcher({m, 10240}).with_dtype<bf16_t>().with_device(dev).verify(normed);
TensorMatcher({4, 10240}).with_dtype<bf16_t>().with_device(dev).verify(weight);
TensorMatcher({m, 8, 4}).with_dtype<fp32_t>().with_device(dev).verify(partials);
const HcCombineSplitParams params{
.block_output = nullptr,
.residual = nullptr,
.normed_residual = normed.data_ptr(),
.inject_weight = weight.data_ptr(),
.output = nullptr,
.partials = static_cast<float*>(partials.data_ptr())};
LaunchKernel(dim3(m.unwrap(), 32, 1), 32, dev.unwrap())
.enable_pdl(true)(hc_combine_gate_kernel<4, 2560, true, bf16_t>, params);
}
static void apply(
tvm::ffi::TensorView block,
tvm::ffi::TensorView residual,
tvm::ffi::TensorView partials,
tvm::ffi::TensorView out) {
using namespace host;
SymbolicSize m{"rows"};
SymbolicDevice dev;
dev.set_options<kDLCUDA>();
TensorMatcher({m, 2560}).with_dtype<bf16_t>().with_device(dev).verify(block);
TensorMatcher({m, 10240}).with_dtype<bf16_t>().with_device(dev).verify(residual).verify(out);
TensorMatcher({m, 8, 4}).with_dtype<fp32_t>().with_device(dev).verify(partials);
const HcCombineSplitParams params{
.block_output = block.data_ptr(),
.residual = residual.data_ptr(),
.normed_residual = nullptr,
.inject_weight = nullptr,
.output = out.data_ptr(),
.partials = static_cast<float*>(partials.data_ptr())};
LaunchKernel(dim3(m.unwrap(), 8, 1), 160, dev.unwrap())
.enable_pdl(true)(hc_combine_apply_kernel<4, 2560, true, bf16_t>, params);
}
};
struct HcCombineNormDecodeParams {
const bf16_t* block;
const bf16_t* residual;
const float* partials;
const bf16_t* weight;
bf16_t* combined;
bf16_t* normalized;
float eps;
};
template <int kParts>
__global__
__launch_bounds__(160) void hc_combine_norm_decode_kernel(const HcCombineNormDecodeParams __grid_constant__ params) {
using namespace device;
using BFloat16x2 = packed_t<bf16_t>;
using Storage = AlignedVector<BFloat16x2, 8>;
constexpr uint32_t kThreads = 160;
constexpr uint32_t kWarps = 5;
static_assert(kThreads % kParts == 0);
const uint32_t row = blockIdx.x / (4 * kParts);
const uint32_t branch = (blockIdx.x / kParts) % 4;
const uint32_t part = blockIdx.x % kParts;
const uint32_t tid = threadIdx.x;
const auto gmem = tile::Memory<Storage>::cta(kThreads);
const auto residual = params.residual + (row * 4 + branch) * 2560;
const auto block_output = params.block + row * 2560;
const auto norm_weight = params.weight + branch * 2560;
__shared__ float warp_sums[kWarpThreads];
PDLWaitPrimary<true>();
float total = 0.0f;
#pragma unroll
for (uint32_t split = 0; split < 8; ++split)
total += params.partials[(row * 8 + split) * 4 + branch];
const float gate = 2.0f / (1.0f + math::exp(-total / 4));
const auto residual_values = gmem.load(residual, 0);
const auto block_values = gmem.load(block_output, 0);
const auto weight_values = gmem.load(norm_weight, 0);
Storage combined;
#pragma unroll
for (uint32_t i = 0; i < 8; ++i) {
const auto [rx, ry] = cast<fp32x2_t>(residual_values[i]);
const auto [yx, yy] = cast<fp32x2_t>(block_values[i]);
// Match the stock apply's FMA and its BF16 store before normalization.
combined[i] = cast<BFloat16x2>(fp32x2_t{rx + gate * yx, ry + gate * yy});
}
float sum = 0.0f;
#pragma unroll
for (uint32_t i = 0; i < 8; ++i) {
const auto [x, y] = cast<fp32x2_t>(combined[i]);
sum += x * x + y * y;
}
sum = warp::reduce_sum(sum);
const auto warp_id = tid / kWarpThreads;
warp_sums[warp_id] = sum;
__syncthreads();
if (warp_id == 0) {
const auto local_sum = tid < kWarps ? warp_sums[tid] : 0.0f;
sum = warp::reduce_sum(local_sum);
warp_sums[tid] = math::rsqrt(sum / 2560 + params.eps);
}
__syncthreads();
const float inv_rms = warp_sums[warp_id];
if (tid >= part * (kThreads / kParts) && tid < (part + 1) * (kThreads / kParts)) {
Storage normalized;
#pragma unroll
for (uint32_t i = 0; i < 8; ++i) {
const auto [x, y] = cast<fp32x2_t>(combined[i]);
const auto [wx, wy] = cast<fp32x2_t>(weight_values[i]);
normalized[i] = cast<BFloat16x2>(fp32x2_t{x * inv_rms * (1.0f + wx), y * inv_rms * (1.0f + wy)});
}
gmem.store(params.combined + (row * 4 + branch) * 2560, combined, 0);
gmem.store(params.normalized + (row * 4 + branch) * 2560, normalized, 0);
}
PDLTriggerSecondary<true>();
}
template <int kParts>
struct HcCombineNormDecode {
static void
run(tvm::ffi::TensorView block,
tvm::ffi::TensorView residual,
tvm::ffi::TensorView partials,
tvm::ffi::TensorView weight,
tvm::ffi::TensorView combined,
tvm::ffi::TensorView normalized,
float eps) {
using namespace host;
SymbolicSize m{"rows"};
SymbolicDevice dev;
dev.set_options<kDLCUDA>();
TensorMatcher({m, 2560}).with_dtype<bf16_t>().with_device(dev).verify(block);
TensorMatcher({m, 10240})
.with_dtype<bf16_t>()
.with_device(dev)
.verify(residual)
.verify(combined)
.verify(normalized);
TensorMatcher({10240}).with_dtype<bf16_t>().with_device(dev).verify(weight);
TensorMatcher({m, 8, 4}).with_dtype<fp32_t>().with_device(dev).verify(partials);
CHECK_HOST(m.unwrap() > 0 && m.unwrap() <= 4);
const HcCombineNormDecodeParams p{
static_cast<const bf16_t*>(block.data_ptr()),
static_cast<const bf16_t*>(residual.data_ptr()),
static_cast<const float*>(partials.data_ptr()),
static_cast<const bf16_t*>(weight.data_ptr()),
static_cast<bf16_t*>(combined.data_ptr()),
static_cast<bf16_t*>(normalized.data_ptr()),
eps};
LaunchKernel(m.unwrap() * 4 * kParts, 160, dev.unwrap()).enable_pdl(true)(hc_combine_norm_decode_kernel<kParts>, p);
}
};
} // namespace sglang
@@ -359,7 +359,18 @@ def fused_recurrent_gated_delta_rule_packed_decode(
raise ValueError(
f"Packed decode kernel only supports NK=1 (got K={K}, BK={BK})."
)
BV = min(triton.next_power_of_2(V), 32)
# Single-token TP4 decode otherwise launches only 48 CTAs and uses over
# 200 registers per thread. Smaller value tiles expose 192 independent
# CTAs without changing the recurrence or its one-warp K reductions.
use_small_value_tile = (
B == 1
and qkv_dim == 2560
and mixed_qkv.dtype == torch.bfloat16
and initial_state.dtype == torch.float32
and (HV, V, K) == (12, 128, 128)
and torch.cuda.get_device_capability(mixed_qkv.device)[0] == 10
)
BV = min(triton.next_power_of_2(V), 8 if use_small_value_tile else 32)
num_stages = 3
num_warps = 1
@@ -10,6 +10,16 @@ from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
__all__ = []
register_kernel(
KernelSpec(
op="communication.moe_finalize_shared_gate_all_reduce",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.communication.all_reduce_fusion:moe_finalize_shared_gate_all_reduce",
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}),
description="BF16 routed finalize, FP32 shared gate, and clustered all-reduce.",
)
)
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
for _mod, _fn in [
@@ -1,6 +1,6 @@
"""Fused deferred-MoE finalize + 1shot push all-reduce [+ RMSNorm] (bf16).
One entry point, :func:`moe_finalize_all_reduce`, over
The base entry point, :func:`moe_finalize_all_reduce`, wraps
``csrc/distributed/all_reduce_fusion.cuh``::
out[t] = allreduce( sum_k expert_weights[t, k] * gemm2_out[idx[t*top_k + k]]
@@ -14,8 +14,10 @@ materializes; ``idx == -1`` slots (EP: non-local expert) contribute nothing.
Small-batch only: the whole ``[T, hidden]`` bf16 row view must fit one push
slot (checked C++-side; :func:`fits_push_slot` lets callers pre-check).
Needs :func:`register_comm` once per process (the CustomAllReduceV2
``Communicator``); the ops key on ``world_size`` alone.
The base entry point needs :func:`register_comm` once per process and keys on
``world_size`` alone. :func:`moe_finalize_shared_gate_all_reduce` takes the
caller's communicator directly and applies an FP32 shared gate after rounding
the routed values to BF16, before the collective.
"""
from __future__ import annotations
@@ -229,3 +231,70 @@ def moe_finalize_all_reduce(
prefetch_metadata,
)
return out
@cache_once
def _jit_shared_gate_module(
world_size: int,
hidden_dim: int,
top_k: int,
cluster_size: int,
weight_dtype: torch.dtype,
) -> Module:
require_cluster_launch_arch()
assert cluster_size in valid_cluster_sizes(hidden_dim)
args = make_cpp_args(
world_size,
hidden_dim,
top_k,
cluster_size,
is_arch_support_pdl(),
weight_dtype,
False, # kMhc
False, # kQuant
False, # kCollapse
True, # kGatedShared
)
return load_jit(
"moe_finalize_shared_gate_all_reduce",
*args,
cuda_files=["distributed/all_reduce_fusion.cuh"],
cuda_wrappers=[("run", f"MoeFinalizeAllReduceKernel<{args}>::run_shared_gate")],
)
def moe_finalize_shared_gate_all_reduce(
gemm2_out: torch.Tensor,
expanded_idx_to_permuted_idx: torch.Tensor,
expert_weights: torch.Tensor,
shared_output: torch.Tensor,
shared_gate: torch.Tensor,
comm: Communicator,
*,
cluster_size: Optional[int] = None,
out: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Finalize routed BF16 values, gate the shared branch, then all-reduce.
``shared_gate`` is FP32 and is applied after the routed BF16 rounding.
The caller owns the communicator and joins all input-producing streams.
"""
_, hidden_dim = shared_output.shape
if out is None:
out = torch.empty_like(shared_output)
_jit_shared_gate_module(
comm.world_size,
hidden_dim,
expert_weights.shape[1],
cluster_size or default_cluster_size(hidden_dim),
expert_weights.dtype,
).run(
comm,
out,
gemm2_out,
expanded_idx_to_permuted_idx,
expert_weights,
shared_output,
shared_gate,
)
return out
@@ -13,6 +13,18 @@ from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
__all__ = []
for _fn in ("hc_combine_gate", "hc_combine_apply", "hc_combine_apply_norm"):
register_kernel(
KernelSpec(
op=f"elementwise.{_fn}",
backend=KernelBackend.JIT,
target=f"sglang.kernels.ops.elementwise.hc_combine_decode:{_fn}",
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}),
description="BF16 HC decode with four branches of width 2560 and PDL.",
)
)
del _fn
# Public entry points inventoried by logical operator group (RFC #29630).
register_kernel(
@@ -488,6 +488,7 @@ def _fused_gate_sigmoid_mul_add_kernel(
BLOCK_SIZE: tl.constexpr,
DO_ADD: tl.constexpr = True,
USE_PDL: tl.constexpr = False,
GATE_ONLY: tl.constexpr = False,
):
pid = tl.program_id(axis=0).to(tl.int64)
row_offset = pid * hidden_dim
@@ -503,23 +504,30 @@ def _fused_gate_sigmoid_mul_add_kernel(
h = tl.load(hidden_states_ptr + row_offset + offsets, mask=mask, other=0.0).to(
tl.float32
)
s = tl.load(shared_output_ptr + row_offset + offsets, mask=mask, other=0.0).to(
tl.float32
)
if DO_ADD:
f = tl.load(output_ptr + row_offset + offsets, mask=mask, other=0.0).to(
if not GATE_ONLY:
s = tl.load(shared_output_ptr + row_offset + offsets, mask=mask, other=0.0).to(
tl.float32
)
if DO_ADD:
f = tl.load(output_ptr + row_offset + offsets, mask=mask, other=0.0).to(
tl.float32
)
if USE_PDL:
tl.extra.cuda.gdc_launch_dependents()
if USE_PDL:
tl.extra.cuda.gdc_launch_dependents()
gate_val = tl.sigmoid(tl.sum(h * w, axis=0))
result = gate_val * s
if DO_ADD:
result += f
tl.store(output_ptr + row_offset + offsets, result, mask=mask)
if GATE_ONLY:
# Preserve FP32 for the later shared-add FMA, without an intermediate
# BF16 gated shared output. Keep the standalone gate's PDL ordering.
if USE_PDL:
tl.extra.cuda.gdc_launch_dependents()
tl.store(output_ptr + pid, gate_val)
else:
result = gate_val * s
if DO_ADD:
result += f
tl.store(output_ptr + row_offset + offsets, result, mask=mask)
def _launch_fused_gate_sigmoid_mul(
@@ -0,0 +1,53 @@
"""Separate HC gate preparation and combine, with optional next-branch norm.
The decode specialization accepts four branches of 2560 BF16 values. Gate
partials belong to each invocation, so overlapping sublayers never reuse them.
"""
import torch
from sglang.kernels.jit.utils import cache_once, load_jit
@cache_once
def _jit_module():
return load_jit(
"hc_combine_decode",
cuda_files=["elementwise/hc_combine_decode.cuh"],
cuda_wrappers=[
("gate", "HcCombineDecode::gate"),
("apply", "HcCombineDecode::apply"),
("apply_norm", "HcCombineNormDecode<2>::run"),
],
)
def hc_combine_gate(normed: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
partials = torch.empty(
(normed.shape[0], 8, 4), device=normed.device, dtype=torch.float32
)
_jit_module().gate(normed, weight, partials)
return partials
def hc_combine_apply(
block: torch.Tensor, residual: torch.Tensor, partials: torch.Tensor
) -> torch.Tensor:
combined = torch.empty_like(residual)
_jit_module().apply(block, residual, partials, combined)
return combined
def hc_combine_apply_norm(
block: torch.Tensor,
residual: torch.Tensor,
partials: torch.Tensor,
weight: torch.Tensor,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
combined = torch.empty_like(residual)
normalized = torch.empty_like(residual)
_jit_module().apply_norm(
block, residual, partials, weight, combined, normalized, eps
)
return combined, normalized
+10
View File
@@ -19,6 +19,16 @@ if TYPE_CHECKING:
_CUDA = frozenset({CapabilityRequirement.CUDA})
_HIP = frozenset({CapabilityRequirement.HIP})
register_kernel(
KernelSpec(
op="moe.shared_expert_gate",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.moe.shared_expert_gate:shared_expert_gate",
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}),
description="FP32 sigmoid gate from a single BF16 row of width 2560, with PDL.",
)
)
register_kernel(
KernelSpec(
op="moe.moe_align_block_size",
@@ -0,0 +1,35 @@
"""FP32 sigmoid gate, preserving the shared-expert fused-add reduction."""
import torch
from sglang.kernels.ops.elementwise.elementwise import (
_fused_gate_sigmoid_mul_add_kernel,
)
def shared_expert_gate(
hidden: torch.Tensor, weight: torch.Tensor, out: torch.Tensor | None = None
) -> torch.Tensor:
assert hidden.is_cuda and weight.device == hidden.device
assert hidden.shape == (1, 2560) and hidden.is_contiguous()
assert hidden.dtype == weight.dtype == torch.bfloat16
assert weight.numel() == 2560 and weight.is_contiguous()
gate = out
if gate is None:
gate = torch.empty(hidden.shape[0], device=hidden.device, dtype=torch.float32)
assert gate.shape == (hidden.shape[0],) and gate.dtype == torch.float32
assert gate.device == hidden.device and gate.is_contiguous()
_fused_gate_sigmoid_mul_add_kernel[(hidden.shape[0],)](
hidden,
weight,
None,
gate,
hidden_dim=2560,
BLOCK_SIZE=4096,
DO_ADD=False,
USE_PDL=True,
GATE_ONLY=True,
num_warps=16,
launch_pdl=True,
)
return gate
+95 -6
View File
@@ -1,3 +1,4 @@
from functools import lru_cache
from typing import Optional
import msgspec
@@ -8,6 +9,11 @@ import torch.nn.functional as F
from sglang.kernels.ops.gemm.hc_mix import fused_hc_mix, fused_hc_mix_supported
@lru_cache(None)
def _hc_gate_stream(device: int) -> torch.cuda.Stream:
return torch.cuda.Stream(device=device)
class HyperConnectionConfig(msgspec.Struct, frozen=True):
hc_count: int = 4
hidden_size: int = 64
@@ -159,6 +165,7 @@ class GatedResidual(HyperConnectionBase):
)
self._mix_up_weight_padded = None
self._gate_stream = None
if use_combine:
self.block_inject_weight = nn.Linear(
self.hidden_size * self.hc_count,
@@ -177,6 +184,16 @@ class GatedResidual(HyperConnectionBase):
and vecs % (8 * 160) == 0
and (self.hidden_size // 8) % (vecs // 8) == 0
)
if (
self.hc_count == 4
and self.hidden_size == 2560
and config.hc_per_branch_norm
and config.params_dtype == torch.bfloat16
and torch.cuda.is_available()
and torch.cuda.get_device_capability()[0] == 10
):
# Create before graph capture; all layers on a device share it.
self._gate_stream = _hc_gate_stream(torch.cuda.current_device())
def _mix_compute(
hyper_input_normed: torch.Tensor,
@@ -216,15 +233,23 @@ class GatedResidual(HyperConnectionBase):
self._mix_compute = torch.compile(_mix_compute)
self._combine_compute = torch.compile(_combine_compute)
def mix(self, hyper_input: torch.Tensor):
def mix(
self,
hyper_input: torch.Tensor,
normalized_input: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]]:
assert hyper_input.shape[-1] == self.hc_count * self.hidden_size
if hyper_input.shape[0] == 0:
mixed_input = hyper_input.new_empty(
(*hyper_input.shape[:-1], self.hidden_size), dtype=self.params_dtype
)
return mixed_input, (hyper_input, hyper_input)
return mixed_input, (hyper_input, hyper_input, None)
if self.config.hc_per_branch_norm:
if normalized_input is not None:
assert self.config.hc_per_branch_norm
assert normalized_input.shape == hyper_input.shape
hyper_input_normed = normalized_input
elif self.config.hc_per_branch_norm:
hyper_input_normed = self.hc_norm(hyper_input)
else:
hyper_input_normed = self.hc_norm(
@@ -272,10 +297,74 @@ class GatedResidual(HyperConnectionBase):
self.hc_count,
self.hidden_size,
).to(self.params_dtype)
return mixed_input, (hyper_input, hyper_input_normed)
partials = None
stream = self._gate_stream
if (
stream is not None
and hyper_input.shape[0] == 1
and hyper_input.dtype == torch.bfloat16
and not torch.compiler.is_compiling()
):
from sglang.kernels.ops.elementwise.hc_combine_decode import hc_combine_gate
def combine(self, block_output: torch.Tensor, residuals) -> torch.Tensor:
hyper_input, hyper_input_normed = residuals
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
partials = hc_combine_gate(
hyper_input_normed, self.block_inject_weight.weight
)
hyper_input_normed.record_stream(stream)
return mixed_input, (hyper_input, hyper_input_normed, partials)
def combine_and_normalize(
self,
block_output: torch.Tensor,
residuals: tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]],
next_norm: Optional[GroupedGemmaRMSNorm],
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
"""Return the residual and its next HC normalization explicitly.
The BF16 combine result is retained before calculating the next norm.
Callers must only pass the norm when no PLE or other update intervenes.
"""
hyper_input, _, partials = residuals
if (
partials is not None
and next_norm is not None
and next_norm.group_size == self.hidden_size
and next_norm.weight.dtype == torch.bfloat16
and block_output.dtype == torch.bfloat16
):
from sglang.kernels.ops.elementwise.hc_combine_decode import (
hc_combine_apply_norm,
)
current = torch.cuda.current_stream()
current.wait_stream(self._gate_stream)
partials.record_stream(current)
return hc_combine_apply_norm(
block_output,
hyper_input,
partials,
next_norm.weight,
next_norm.variance_epsilon,
)
return self.combine(block_output, residuals), None
def combine(
self,
block_output: torch.Tensor,
residuals: tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]],
) -> torch.Tensor:
hyper_input, hyper_input_normed, partials = residuals
if partials is not None:
from sglang.kernels.ops.elementwise.hc_combine_decode import (
hc_combine_apply,
)
current = torch.cuda.current_stream()
current.wait_stream(self._gate_stream)
partials.record_stream(current)
return hc_combine_apply(block_output, hyper_input, partials)
assert hyper_input.shape[-1] == self.hc_count * self.hidden_size
assert block_output.shape[-1] == self.hidden_size
if block_output.shape[0] == 0:
@@ -100,10 +100,14 @@ class ExitPolicy:
hidden_states: torch.Tensor,
stream: ResidualStream,
forward_batch: ForwardBatch,
*,
already_reduced: bool = False,
):
"""Complete the output of the operation-scheduled producer path, which
never defers: the sum it owes, then its move and write-back."""
steps = self.plan.path_for(forward_batch)
if already_reduced and steps.output_move_completes_sum:
raise ValueError("already-reduced output cannot use a reduce-scatter move")
hidden_states, residual = self._complete_now(
hidden_states,
stream.residual,
@@ -111,7 +115,8 @@ class ExitPolicy:
dp_step=None,
steps=steps,
output_move=steps.output_move,
owes_sum=steps.output.group is not None
owes_sum=not already_reduced
and steps.output.group is not None
and not steps.output_move_completes_sum,
)
return self._record_output(
@@ -29,11 +29,11 @@ class GatedResidualState:
per-stream gates, and writes its output back scaled by an injection
coefficient computed from that same normalized residual.
Unlike the hyper-connection streams in mhc.py and ihc.py, whose read
produces the coefficient its write-back consumes, the coefficient here is
computed at write time. What has to survive from a stage's read to its
write-back is therefore the normalized residual itself, which this state
holds. Parameters belong to the owning layer's modules.
The read carries its normalized residual to write-back. Decode may also
prepare gate partials during the read; otherwise write-back computes the
coefficient from that normalized residual. Both follow the residual rows
through any slicing and are cleared after the FFN write-back.
Parameters belong to the owning layer's modules.
Args:
expand: Widens the layer stack's input into the stream representation,
@@ -56,6 +56,7 @@ class GatedResidualState:
attn_reads_every_row: bool = False
# Produced by a stage's read and consumed by that same stage's write-back.
normed: Optional[torch.Tensor] = None
gate_partials: Optional[torch.Tensor] = None
def _read(self, mix, residual, out_norm):
if out_norm is not None:
@@ -63,7 +64,8 @@ class GatedResidualState:
"a gated hyper-connection read with a separate norm; the read "
"normalizes the streams itself"
)
hidden_states, (residual, self.normed) = mix(residual)
hidden_states, residuals = mix(residual)
residual, self.normed, self.gate_partials = residuals
return hidden_states, residual
def read_attn_input(self, residual, out_norm=None):
@@ -74,18 +76,25 @@ class GatedResidualState:
return self._read(self.ffn_mix, residual, out_norm)
def apply_attn_combine(self, hidden_states, residual):
return self.attn_combine(hidden_states, (residual, self.normed))
return self.attn_combine(
hidden_states, (residual, self.normed, self.gate_partials)
)
def apply_ffn_combine(self, hidden_states, residual):
return self.ffn_combine(hidden_states, (residual, self.normed))
return self.ffn_combine(
hidden_states, (residual, self.normed, self.gate_partials)
)
def clear_coefficients(self):
self.normed = None
self.gate_partials = None
def slice_residual_attn_tp(self, residual):
parallel = get_parallel()
rank, size = parallel.attn_tp_rank, parallel.attn_tp_size
self.normed = self.normed.tensor_split(size)[rank]
if self.gate_partials is not None:
self.gate_partials = self.gate_partials.tensor_split(size)[rank]
return residual.tensor_split(size)[rank]
def gather_residual_attn_tp(self, residual):
@@ -262,12 +262,17 @@ class StageBoundary:
hidden_states = stream.complete(hidden_states)
return hidden_states, stream.snapshot(hidden_states)
def complete_now(self, hidden_states, forward_batch):
"""Complete an FFN output outside exit() (the operation-scheduled TBO
path, which never defers): the sum it owes, then the move onto the
rows the layer hands on."""
def complete_now(self, hidden_states, forward_batch, *, already_reduced=False):
"""Complete an FFN output outside exit(), including its residual update.
already_reduced is for a producer kernel that fused the output sum.
Row movement and residual write-back still belong to this boundary.
"""
return self.plan.output.complete_now(
hidden_states, stream_of(forward_batch), forward_batch
hidden_states,
stream_of(forward_batch),
forward_batch,
already_reduced=already_reduced,
)
def sum_part(self, hidden_states, forward_batch, group: SumGroup):
@@ -509,9 +509,16 @@ class FusedMoE(torch.nn.Module):
and isinstance(self.quant_method, Fp8MoEMethod)
and self.quant_method.block_quant
)
qwen4_bf16_deferred = (
isinstance(self.quant_method, UnquantizedFusedMoEMethod)
and params_dtype == torch.bfloat16
and hidden_size == 2560
and num_experts == 512
and top_k == 10
)
self.supports_deferred_finalize = (
get_moe_runner_backend().is_flashinfer_trtllm()
and (nvfp4_deferred or qwen35_fp8_deferred)
and (nvfp4_deferred or qwen35_fp8_deferred or qwen4_bf16_deferred)
)
global _deferred_finalize_info_logged
if not _deferred_finalize_info_logged:
@@ -1640,6 +1640,9 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
hidden_states = dispatch_output.hidden_states
topk_output = dispatch_output.topk_output
defer_finalize = _deferred_finalize_enabled.get()
if defer_finalize and use_routed_topk:
raise RuntimeError("BF16 deferred finalize requires bypassed TopK")
with use_symmetric_memory(
get_parallel().tp_group, disabled=not is_allocation_symmetric()
@@ -1697,7 +1700,12 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
routed_scaling_factor=runner_config.routed_scaling_factor,
tune_max_num_tokens=next_power_of_2(hidden_states.shape[0]),
activation_type=activation_type,
do_finalize=not defer_finalize,
)
if defer_finalize:
final_hidden_states = _make_deferred_finalize_output(
final_hidden_states, top_k=topk_config.top_k
)
return StandardCombineInput(hidden_states=final_hidden_states)
@@ -0,0 +1,95 @@
"""Single-token BF16 Qwen4 MoE finalize, shared gate and TP4 collective."""
from __future__ import annotations
from functools import lru_cache
from typing import TYPE_CHECKING
import torch
from sglang.kernels.ops.communication.all_reduce_fusion import (
moe_finalize_shared_gate_all_reduce,
)
from sglang.kernels.ops.communication.mp import register_comm_cleanup
from sglang.kernels.ops.moe.shared_expert_gate import shared_expert_gate
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
CustomAllReduceV2,
)
from sglang.srt.layers.dp_attention import is_enable_moe_cp_allgather
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.runtime_context import get_exec, get_lora, get_parallel, get_spec
if TYPE_CHECKING:
from sglang.srt.models.qwen2_moe import Qwen2MoeSparseMoeBlock
@lru_cache(None)
def _get_decode_comm(tp_comm: CustomAllReduceV2) -> CustomAllReduceV2 | None:
comm = CustomAllReduceV2(
tp_comm.group,
tp_comm.device,
max_pull_size=0,
max_pull_blocks=0,
max_push_size=4 * 1024 * 1024,
max_push_blocks=512,
)
if comm.disabled:
return None
register_comm_cleanup(comm)
return comm
def prepare_qwen4_decode_comm(
mlp: Qwen2MoeSparseMoeBlock,
) -> CustomAllReduceV2 | None:
parallel = get_parallel()
if (
parallel.tp_size != 4
or parallel.attn_dp_size != 1
or parallel.moe_ep_size != 1
or get_lora().enable_lora
or get_spec().speculative_algorithm is not None
or get_exec().overlap.enable_two_batch_overlap
or is_enable_moe_cp_allgather()
or not mlp.supports_deferred_finalize
or mlp.num_experts != 512
or mlp.experts.w13_weight.dtype != torch.bfloat16
or mlp.shared_expert_gate is None
or mlp.enable_shared_expert_fusion
or not torch.cuda.is_available()
or torch.cuda.get_device_capability()[0] != 10
):
return None
tp_comm = parallel.tp_group.ca_comm
if not isinstance(tp_comm, CustomAllReduceV2) or tp_comm.disabled:
return None
return _get_decode_comm(tp_comm)
def qwen4_decode_moe(
hidden: torch.Tensor, mlp: Qwen2MoeSparseMoeBlock, comm: CustomAllReduceV2
) -> torch.Tensor:
if mlp.alt_stream is not None and get_is_capture_mode():
current = torch.cuda.current_stream()
mlp.alt_stream.wait_stream(current)
# FC2 stays on the consuming stream for its PDL dependency. Both
# branches read the input and write independently allocated outputs.
with torch.cuda.stream(mlp.alt_stream):
gate = shared_expert_gate(hidden, mlp.shared_expert_gate.weight)
shared = mlp._forward_shared_experts(hidden, apply_gate=False)
deferred = mlp._forward_router_experts(hidden, defer_finalize=True)
current.wait_stream(mlp.alt_stream)
gate.record_stream(current)
shared.record_stream(current)
else:
gate = shared_expert_gate(hidden, mlp.shared_expert_gate.weight)
shared = mlp._forward_shared_experts(hidden, apply_gate=False)
deferred = mlp._forward_router_experts(hidden, defer_finalize=True)
return moe_finalize_shared_gate_all_reduce(
deferred.gemm2_out,
deferred.expanded_idx_to_permuted_idx,
deferred.expert_weights,
shared,
gate,
comm.obj,
)
+117 -11
View File
@@ -1,7 +1,8 @@
"""Inference-only Qwen4-Exp (text + VL) on the Qwen3.5 backbone."""
import math
from contextlib import nullcontext
from contextlib import contextmanager, nullcontext
from dataclasses import dataclass
from typing import Any, Iterable, Optional, Set, Tuple, Union
import msgspec
@@ -34,6 +35,7 @@ from sglang.srt.layers.dp_attention import (
)
from sglang.srt.layers.hyperconnection import (
GatedResidual,
GroupedGemmaRMSNorm,
HyperConnectionConfig,
)
from sglang.srt.layers.layer_boundary import (
@@ -47,6 +49,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.modelopt_quant import (
ModelOptMixedPrecisionConfig,
@@ -1422,11 +1425,24 @@ class _MixStreams:
"""The layer stack's terminal read: the gated mix down to one stream. The
stack ends on the mix, with no final norm after it."""
def __init__(self, hyper_connection_mixer):
def __init__(self, hyper_connection_mixer, normalized_input=None):
self.hyper_connection_mixer = hyper_connection_mixer
self.normalized_input = normalized_input
def __call__(self, hidden_states):
return self.hyper_connection_mixer.mix(hidden_states)[0]
return self.hyper_connection_mixer.mix(
hidden_states, normalized_input=self.normalized_input
)[0]
@dataclass
class _HCNorms:
"""Prepared norms for one layer invocation; never carried across PLE or PP."""
attn_input: Optional[torch.Tensor]
next_norm: Optional[GroupedGemmaRMSNorm]
ffn_input: Optional[torch.Tensor] = None
output: Optional[torch.Tensor] = None
def _has_ple(layer_id, config) -> bool:
@@ -1470,6 +1486,7 @@ class Qwen4ExpLayerExtensionMixin:
self.ple = None
self._ple_forward_batch = None
self._ple_batch = None
self._hc_norms = None
# The gated reads normalize the streams themselves, so these layers
# have neither of the norms their base class builds.
@@ -1515,15 +1532,41 @@ class Qwen4ExpLayerExtensionMixin:
GatedResidualState(
expand=self._widen_streams,
attn_mix=self._attn_mix,
ffn_mix=self.mlp_hyper_connection.mix,
attn_combine=self.attn_hyper_connection.combine,
ffn_combine=self.mlp_hyper_connection.combine,
ffn_mix=self._ffn_mix,
attn_combine=self._attn_combine,
ffn_combine=self._ffn_combine,
# The PLE embedding is computed for every row.
attn_reads_every_row=self.ple is not None,
).residual_ops(),
sparse=isinstance(self.mlp, Qwen2MoeSparseMoeBlock),
)
from sglang.srt.layers.moe.qwen4_decode import prepare_qwen4_decode_comm
self._decode_moe_comm = (
prepare_qwen4_decode_comm(self.mlp)
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock)
else None
)
@contextmanager
def hc_norms(self, normalized_input, next_norm):
norms = _HCNorms(normalized_input, next_norm)
# A prepared norm can only follow its residual without row movement.
self._hc_norms = (
norms
if get_parallel().attn_dp_size == 1
and get_parallel().attn_cp_size == 1
and get_parallel().moe_ep_size == 1
and not get_attn_tp_context().input_scattered
and get_moe_a2a_backend().is_none()
else None
)
try:
yield norms
finally:
self._hc_norms = None
def _widen_streams(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Open the layer stack's input into one stream per hyper connection.
An MTP draft is handed streams that are already widened."""
@@ -1538,7 +1581,42 @@ class Qwen4ExpLayerExtensionMixin:
own contribution to the streams and joins them before the read."""
if self.ple is not None:
residual = self._add_ple_embedding(residual)
return self.attn_hyper_connection.mix(residual)
normalized_input = (
self._hc_norms.attn_input
if self._hc_norms is not None and self.ple is None
else None
)
return self.attn_hyper_connection.mix(
residual, normalized_input=normalized_input
)
def _attn_combine(self, hidden_states, residuals):
if self._hc_norms is None:
return self.attn_hyper_connection.combine(hidden_states, residuals)
hidden_states, self._hc_norms.ffn_input = (
self.attn_hyper_connection.combine_and_normalize(
hidden_states, residuals, self.mlp_hyper_connection.hc_norm
)
)
return hidden_states
def _ffn_mix(self, residual):
normalized_input = (
self._hc_norms.ffn_input if self._hc_norms is not None else None
)
return self.mlp_hyper_connection.mix(
residual, normalized_input=normalized_input
)
def _ffn_combine(self, hidden_states, residuals):
if self._hc_norms is None:
return self.mlp_hyper_connection.combine(hidden_states, residuals)
hidden_states, self._hc_norms.output = (
self.mlp_hyper_connection.combine_and_normalize(
hidden_states, residuals, self._hc_norms.next_norm
)
)
return hidden_states
def _add_ple_embedding(self, residual: torch.Tensor) -> torch.Tensor:
forward_batch, ple_batch = self._ple_forward_batch, self._ple_batch
@@ -1573,6 +1651,21 @@ class Qwen4ExpLayerExtensionMixin:
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
if (
self._decode_moe_comm is not None
and hidden_states.shape[0] == 1
and hidden_states.dtype == torch.bfloat16
and not get_attn_tp_context().input_scattered
and not torch.compiler.is_compiling()
):
from sglang.srt.layers.moe.qwen4_decode import qwen4_decode_moe
hidden_states = qwen4_decode_moe(
hidden_states, self.mlp, self._decode_moe_comm
)
return self.ffn_boundary.complete_now(
hidden_states, forward_batch, already_reduced=True
)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock):
hidden_states = self.mlp(
@@ -1855,8 +1948,6 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
if self.pp_group.is_last_rank
else PPMissingLayer()
)
# The stack's terminal read mixes the streams down; there is no final norm.
self._mix_streams = _MixStreams(self.hyper_connection_mixer)
def forward(
self,
@@ -1903,6 +1994,7 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
ngram_eos_token_id=self.ple_ngram_eos_token_id,
)
aux_hidden_states = AuxHiddenStateList()
normalized_input = None
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
if i + 1 < self.end_layer:
@@ -1914,7 +2006,17 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
next_ple.start_prefetch(
batch=ple_batch, forward_batch=forward_batch
)
with get_global_expert_distribution_recorder().with_current_layer(i):
next_norm = None
if i + 1 < self.end_layer:
next_layer = self.layers[i + 1]
if next_layer.ple is None:
next_norm = next_layer.attn_hyper_connection.hc_norm
elif self.pp_group.is_last_rank:
next_norm = self.hyper_connection_mixer.hc_norm
with (
get_global_expert_distribution_recorder().with_current_layer(i),
layer.hc_norms(normalized_input, next_norm) as norms,
):
hidden_states = layer(
positions=positions,
hidden_states=hidden_states,
@@ -1927,6 +2029,8 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
),
)
normalized_input = norms.output
if breakable_ple:
_breakable_commit_ple_batch(batch=ple_batch)
else:
@@ -1942,7 +2046,9 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
hc_hidden_states = hidden_states
hidden_states = residual_batch.final_norm(
hidden_states, forward_batch, self._mix_streams
hidden_states,
forward_batch,
_MixStreams(self.hyper_connection_mixer, normalized_input),
)
if not forward_batch.forward_mode.is_idle():
return hidden_states, hc_hidden_states