diff --git a/python/sglang/kernels/jit/csrc/distributed/all_reduce_fusion.cuh b/python/sglang/kernels/jit/csrc/distributed/all_reduce_fusion.cuh index 123dd2b126..ac2837510d 100644 --- a/python/sglang/kernels/jit/csrc/distributed/all_reduce_fusion.cuh +++ b/python/sglang/kernels/jit/csrc/distributed/all_reduce_fusion.cuh @@ -68,13 +68,14 @@ SGL_DEVICE void barrier_cluster_wait() { template struct MoeFinalizeAllReduceParams { - bf16_t* out; // [num_tokens, kHiddenDim], output-only - const bf16_t* gemm2; // [P, kHiddenDim], permuted / padded rows - const int32_t* idx; // [num_tokens * kTopK], -1 = dropped slot - const WeightT* weights; // [num_tokens, kTopK], scaling already folded in - const bf16_t* shared; // [num_tokens, kHiddenDim] (kHasShared only) - const bf16_t* norm_weight; // [kHiddenDim] (kNorm only) - float norm_eps; // kNorm only + bf16_t* out; // [num_tokens, kHiddenDim], output-only + const bf16_t* gemm2; // [P, kHiddenDim], permuted / padded rows + const int32_t* idx; // [num_tokens * kTopK], -1 = dropped slot + const WeightT* weights; // [num_tokens, kTopK], scaling already folded in + const bf16_t* shared; // [num_tokens, kHiddenDim] (kHasShared only) + const float* shared_gate = nullptr; // Optional FP32 sigmoid, one per token. + const bf16_t* norm_weight; // [kHiddenDim] (kNorm only) + float norm_eps; // kNorm only // Caller's promise that everything read before the PDL wait is complete when // the predecessor merely *triggers*: no all-reduce on this plane right before // it, and the routing metadata's producers finished (PDL completion is not @@ -199,7 +200,14 @@ struct RowClusterTrait { // --- stage 1: the deferred finalize of one 16B vector ------------------------ // The shared-expert vector is loaded first so that load is in flight while the // routing rows and the kTopK gathers are fetched. -template +template < + uint32_t kHiddenDim, + uint32_t kTopK, + bool kHasShared, + bool kUsePDL, + uint32_t kWorldSize, + typename WeightT, + bool kGatedShared = false> SGL_DEVICE StageVec finalize_vec(const MoeFinalizeAllReduceParams& params, uint32_t token, uint32_t hvec) { using namespace device; @@ -263,7 +271,12 @@ finalize_vec(const MoeFinalizeAllReduceParams& params, uint if constexpr (kHasShared) { const auto routed = cast(cast(acc[j])); const auto sh = cast(shared_in[j]); - out[j] = cast(fp32x2_t{routed.x + sh.x, routed.y + sh.y}); + if constexpr (kGatedShared) { + const float gate = params.shared_gate[token]; + out[j] = cast(fp32x2_t{fmaf(gate, sh.x, routed.x), fmaf(gate, sh.y, routed.y)}); + } else { + out[j] = cast(fp32x2_t{routed.x + sh.x, routed.y + sh.y}); + } } else { out[j] = cast(acc[j]); } @@ -287,7 +300,8 @@ template < typename WeightT, bool kMhc = false, bool kQuant = false, - bool kCollapse = false> + bool kCollapse = false, + bool kGatedShared = false> __global__ __launch_bounds__(RowClusterTrait::kBlockSize) __cluster_dims__(1, kClusterSize, 1) void moe_finalize_all_reduce_kernel( const __grid_constant__ MoeFinalizeAllReduceParams params) { @@ -342,7 +356,8 @@ __global__ __launch_bounds__(RowClusterTrait::kBlockSi // stage 1: finalize this row's vector in registers and push it to every peer const auto vid = row_idx * kRowVecs + hvec; { - auto vec = finalize_vec(params, row_idx, hvec); + auto vec = + finalize_vec(params, row_idx, hvec); Lamport::clear_pos_zero(vec.data()); #pragma unroll for (uint32_t i = 0; i < kWorldSize; ++i) { @@ -456,7 +471,8 @@ template < typename WeightT, bool kMhc = false, bool kQuant = false, - bool kCollapse = false> + bool kCollapse = false, + bool kGatedShared = false> struct MoeFinalizeAllReduceKernel { private: static_assert(std::is_same_v || std::is_same_v); @@ -476,7 +492,8 @@ struct MoeFinalizeAllReduceKernel { WeightT, kMhc, kQuant, - kCollapse>; + kCollapse, + kGatedShared>; public: /// out = [allreduce over ranks of] finalize(gemm2_out, idx, weights) [+ shared] [-> RMSNorm(norm_weight, eps)]. @@ -509,6 +526,37 @@ struct MoeFinalizeAllReduceKernel { std::nullopt); } + // Preserve routed BF16 rounding, then apply the FP32 shared gate with FMA. + static void run_shared_gate( + CommunicatorRef ref, + TensorView out, + TensorView gemm2_out, + TensorView permuted_idx, + TensorView expert_weights, + TensorView shared_output, + TensorView shared_gate) { + static_assert(kGatedShared && !kMhc); + run_impl( + ref, + out, + gemm2_out, + permuted_idx, + expert_weights, + shared_output, + std::nullopt, + 0.0, + false, + std::nullopt, + std::nullopt, + std::nullopt, + std::nullopt, + std::nullopt, + std::nullopt, + std::nullopt, + std::nullopt, + shared_gate); + } + /// Finalize + all-reduce + HC=4 post; original reduced output is retained. static void run_mhc_post( CommunicatorRef ref, @@ -628,7 +676,8 @@ struct MoeFinalizeAllReduceKernel { std::optional pre = std::nullopt, std::optional normalized = std::nullopt, std::optional quantized = std::nullopt, - std::optional scales = std::nullopt) { + std::optional scales = std::nullopt, + std::optional shared_gate = std::nullopt) { using namespace host; const auto& comm = *ref.get(); const auto& push = comm.get_push_obj(); @@ -664,6 +713,10 @@ struct MoeFinalizeAllReduceKernel { .with_device(device) .verify(shared_output.value()); } + if constexpr (kGatedShared) { + CHECK_HOST(shared_output.has_value() && shared_gate.has_value()); + TensorMatcher({T}).with_dtype().with_device(device).verify(shared_gate.value()); + } if (norm_weight.has_value()) { TensorMatcher({kHiddenDim}) .with_strides({1}) @@ -717,6 +770,7 @@ struct MoeFinalizeAllReduceKernel { .idx = static_cast(permuted_idx.data_ptr()), .weights = static_cast(expert_weights.data_ptr()), .shared = shared_output.has_value() ? static_cast(shared_output.value().data_ptr()) : nullptr, + .shared_gate = shared_gate.has_value() ? static_cast(shared_gate.value().data_ptr()) : nullptr, .norm_weight = norm_weight.has_value() ? static_cast(norm_weight.value().data_ptr()) : nullptr, .norm_eps = static_cast(eps), .prefetch_metadata = prefetch_metadata, diff --git a/python/sglang/kernels/jit/csrc/elementwise/hc_combine_decode.cuh b/python/sglang/kernels/jit/csrc/elementwise/hc_combine_decode.cuh new file mode 100644 index 0000000000..3b9bd482e0 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/elementwise/hc_combine_decode.cuh @@ -0,0 +1,159 @@ +#pragma once +#include "hc_combine.cuh" + +namespace sglang { +// Expose the existing two stock stages separately; all arithmetic is unchanged. +struct HcCombineDecode { + static void gate(tvm::ffi::TensorView normed, tvm::ffi::TensorView weight, tvm::ffi::TensorView partials) { + using namespace host; + SymbolicSize m{"rows"}; + SymbolicDevice dev; + dev.set_options(); + TensorMatcher({m, 10240}).with_dtype().with_device(dev).verify(normed); + TensorMatcher({4, 10240}).with_dtype().with_device(dev).verify(weight); + TensorMatcher({m, 8, 4}).with_dtype().with_device(dev).verify(partials); + const HcCombineSplitParams params{ + .block_output = nullptr, + .residual = nullptr, + .normed_residual = normed.data_ptr(), + .inject_weight = weight.data_ptr(), + .output = nullptr, + .partials = static_cast(partials.data_ptr())}; + LaunchKernel(dim3(m.unwrap(), 32, 1), 32, dev.unwrap()) + .enable_pdl(true)(hc_combine_gate_kernel<4, 2560, true, bf16_t>, params); + } + static void apply( + tvm::ffi::TensorView block, + tvm::ffi::TensorView residual, + tvm::ffi::TensorView partials, + tvm::ffi::TensorView out) { + using namespace host; + SymbolicSize m{"rows"}; + SymbolicDevice dev; + dev.set_options(); + TensorMatcher({m, 2560}).with_dtype().with_device(dev).verify(block); + TensorMatcher({m, 10240}).with_dtype().with_device(dev).verify(residual).verify(out); + TensorMatcher({m, 8, 4}).with_dtype().with_device(dev).verify(partials); + const HcCombineSplitParams params{ + .block_output = block.data_ptr(), + .residual = residual.data_ptr(), + .normed_residual = nullptr, + .inject_weight = nullptr, + .output = out.data_ptr(), + .partials = static_cast(partials.data_ptr())}; + LaunchKernel(dim3(m.unwrap(), 8, 1), 160, dev.unwrap()) + .enable_pdl(true)(hc_combine_apply_kernel<4, 2560, true, bf16_t>, params); + } +}; +struct HcCombineNormDecodeParams { + const bf16_t* block; + const bf16_t* residual; + const float* partials; + const bf16_t* weight; + bf16_t* combined; + bf16_t* normalized; + float eps; +}; + +template +__global__ +__launch_bounds__(160) void hc_combine_norm_decode_kernel(const HcCombineNormDecodeParams __grid_constant__ params) { + using namespace device; + using BFloat16x2 = packed_t; + using Storage = AlignedVector; + constexpr uint32_t kThreads = 160; + constexpr uint32_t kWarps = 5; + static_assert(kThreads % kParts == 0); + const uint32_t row = blockIdx.x / (4 * kParts); + const uint32_t branch = (blockIdx.x / kParts) % 4; + const uint32_t part = blockIdx.x % kParts; + const uint32_t tid = threadIdx.x; + const auto gmem = tile::Memory::cta(kThreads); + const auto residual = params.residual + (row * 4 + branch) * 2560; + const auto block_output = params.block + row * 2560; + const auto norm_weight = params.weight + branch * 2560; + __shared__ float warp_sums[kWarpThreads]; + PDLWaitPrimary(); + float total = 0.0f; +#pragma unroll + for (uint32_t split = 0; split < 8; ++split) + total += params.partials[(row * 8 + split) * 4 + branch]; + const float gate = 2.0f / (1.0f + math::exp(-total / 4)); + const auto residual_values = gmem.load(residual, 0); + const auto block_values = gmem.load(block_output, 0); + const auto weight_values = gmem.load(norm_weight, 0); + Storage combined; +#pragma unroll + for (uint32_t i = 0; i < 8; ++i) { + const auto [rx, ry] = cast(residual_values[i]); + const auto [yx, yy] = cast(block_values[i]); + // Match the stock apply's FMA and its BF16 store before normalization. + combined[i] = cast(fp32x2_t{rx + gate * yx, ry + gate * yy}); + } + float sum = 0.0f; +#pragma unroll + for (uint32_t i = 0; i < 8; ++i) { + const auto [x, y] = cast(combined[i]); + sum += x * x + y * y; + } + sum = warp::reduce_sum(sum); + const auto warp_id = tid / kWarpThreads; + warp_sums[warp_id] = sum; + __syncthreads(); + if (warp_id == 0) { + const auto local_sum = tid < kWarps ? warp_sums[tid] : 0.0f; + sum = warp::reduce_sum(local_sum); + warp_sums[tid] = math::rsqrt(sum / 2560 + params.eps); + } + __syncthreads(); + const float inv_rms = warp_sums[warp_id]; + if (tid >= part * (kThreads / kParts) && tid < (part + 1) * (kThreads / kParts)) { + Storage normalized; +#pragma unroll + for (uint32_t i = 0; i < 8; ++i) { + const auto [x, y] = cast(combined[i]); + const auto [wx, wy] = cast(weight_values[i]); + normalized[i] = cast(fp32x2_t{x * inv_rms * (1.0f + wx), y * inv_rms * (1.0f + wy)}); + } + gmem.store(params.combined + (row * 4 + branch) * 2560, combined, 0); + gmem.store(params.normalized + (row * 4 + branch) * 2560, normalized, 0); + } + PDLTriggerSecondary(); +} + +template +struct HcCombineNormDecode { + static void + run(tvm::ffi::TensorView block, + tvm::ffi::TensorView residual, + tvm::ffi::TensorView partials, + tvm::ffi::TensorView weight, + tvm::ffi::TensorView combined, + tvm::ffi::TensorView normalized, + float eps) { + using namespace host; + SymbolicSize m{"rows"}; + SymbolicDevice dev; + dev.set_options(); + TensorMatcher({m, 2560}).with_dtype().with_device(dev).verify(block); + TensorMatcher({m, 10240}) + .with_dtype() + .with_device(dev) + .verify(residual) + .verify(combined) + .verify(normalized); + TensorMatcher({10240}).with_dtype().with_device(dev).verify(weight); + TensorMatcher({m, 8, 4}).with_dtype().with_device(dev).verify(partials); + CHECK_HOST(m.unwrap() > 0 && m.unwrap() <= 4); + const HcCombineNormDecodeParams p{ + static_cast(block.data_ptr()), + static_cast(residual.data_ptr()), + static_cast(partials.data_ptr()), + static_cast(weight.data_ptr()), + static_cast(combined.data_ptr()), + static_cast(normalized.data_ptr()), + eps}; + LaunchKernel(m.unwrap() * 4 * kParts, 160, dev.unwrap()).enable_pdl(true)(hc_combine_norm_decode_kernel, p); + } +}; +} // namespace sglang diff --git a/python/sglang/kernels/ops/attention/fla/fused_recurrent.py b/python/sglang/kernels/ops/attention/fla/fused_recurrent.py index 9574c7a7b0..44d575fd44 100644 --- a/python/sglang/kernels/ops/attention/fla/fused_recurrent.py +++ b/python/sglang/kernels/ops/attention/fla/fused_recurrent.py @@ -359,7 +359,18 @@ def fused_recurrent_gated_delta_rule_packed_decode( raise ValueError( f"Packed decode kernel only supports NK=1 (got K={K}, BK={BK})." ) - BV = min(triton.next_power_of_2(V), 32) + # Single-token TP4 decode otherwise launches only 48 CTAs and uses over + # 200 registers per thread. Smaller value tiles expose 192 independent + # CTAs without changing the recurrence or its one-warp K reductions. + use_small_value_tile = ( + B == 1 + and qkv_dim == 2560 + and mixed_qkv.dtype == torch.bfloat16 + and initial_state.dtype == torch.float32 + and (HV, V, K) == (12, 128, 128) + and torch.cuda.get_device_capability(mixed_qkv.device)[0] == 10 + ) + BV = min(triton.next_power_of_2(V), 8 if use_small_value_tile else 32) num_stages = 3 num_warps = 1 diff --git a/python/sglang/kernels/ops/communication/__init__.py b/python/sglang/kernels/ops/communication/__init__.py index 6994b74b6e..93b52b67af 100644 --- a/python/sglang/kernels/ops/communication/__init__.py +++ b/python/sglang/kernels/ops/communication/__init__.py @@ -10,6 +10,16 @@ from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec __all__ = [] +register_kernel( + KernelSpec( + op="communication.moe_finalize_shared_gate_all_reduce", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.communication.all_reduce_fusion:moe_finalize_shared_gate_all_reduce", + capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}), + description="BF16 routed finalize, FP32 shared gate, and clustered all-reduce.", + ) +) + # Kernels introduced with Kimi-K3, inventoried by logical operator group. for _mod, _fn in [ diff --git a/python/sglang/kernels/ops/communication/all_reduce_fusion.py b/python/sglang/kernels/ops/communication/all_reduce_fusion.py index 47eff6e12c..6bf17edf0c 100644 --- a/python/sglang/kernels/ops/communication/all_reduce_fusion.py +++ b/python/sglang/kernels/ops/communication/all_reduce_fusion.py @@ -1,6 +1,6 @@ """Fused deferred-MoE finalize + 1shot push all-reduce [+ RMSNorm] (bf16). -One entry point, :func:`moe_finalize_all_reduce`, over +The base entry point, :func:`moe_finalize_all_reduce`, wraps ``csrc/distributed/all_reduce_fusion.cuh``:: out[t] = allreduce( sum_k expert_weights[t, k] * gemm2_out[idx[t*top_k + k]] @@ -14,8 +14,10 @@ materializes; ``idx == -1`` slots (EP: non-local expert) contribute nothing. Small-batch only: the whole ``[T, hidden]`` bf16 row view must fit one push slot (checked C++-side; :func:`fits_push_slot` lets callers pre-check). -Needs :func:`register_comm` once per process (the CustomAllReduceV2 -``Communicator``); the ops key on ``world_size`` alone. +The base entry point needs :func:`register_comm` once per process and keys on +``world_size`` alone. :func:`moe_finalize_shared_gate_all_reduce` takes the +caller's communicator directly and applies an FP32 shared gate after rounding +the routed values to BF16, before the collective. """ from __future__ import annotations @@ -229,3 +231,70 @@ def moe_finalize_all_reduce( prefetch_metadata, ) return out + + +@cache_once +def _jit_shared_gate_module( + world_size: int, + hidden_dim: int, + top_k: int, + cluster_size: int, + weight_dtype: torch.dtype, +) -> Module: + require_cluster_launch_arch() + assert cluster_size in valid_cluster_sizes(hidden_dim) + args = make_cpp_args( + world_size, + hidden_dim, + top_k, + cluster_size, + is_arch_support_pdl(), + weight_dtype, + False, # kMhc + False, # kQuant + False, # kCollapse + True, # kGatedShared + ) + return load_jit( + "moe_finalize_shared_gate_all_reduce", + *args, + cuda_files=["distributed/all_reduce_fusion.cuh"], + cuda_wrappers=[("run", f"MoeFinalizeAllReduceKernel<{args}>::run_shared_gate")], + ) + + +def moe_finalize_shared_gate_all_reduce( + gemm2_out: torch.Tensor, + expanded_idx_to_permuted_idx: torch.Tensor, + expert_weights: torch.Tensor, + shared_output: torch.Tensor, + shared_gate: torch.Tensor, + comm: Communicator, + *, + cluster_size: Optional[int] = None, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """Finalize routed BF16 values, gate the shared branch, then all-reduce. + + ``shared_gate`` is FP32 and is applied after the routed BF16 rounding. + The caller owns the communicator and joins all input-producing streams. + """ + _, hidden_dim = shared_output.shape + if out is None: + out = torch.empty_like(shared_output) + _jit_shared_gate_module( + comm.world_size, + hidden_dim, + expert_weights.shape[1], + cluster_size or default_cluster_size(hidden_dim), + expert_weights.dtype, + ).run( + comm, + out, + gemm2_out, + expanded_idx_to_permuted_idx, + expert_weights, + shared_output, + shared_gate, + ) + return out diff --git a/python/sglang/kernels/ops/elementwise/__init__.py b/python/sglang/kernels/ops/elementwise/__init__.py index 0fa42400cb..a0c3aecab7 100644 --- a/python/sglang/kernels/ops/elementwise/__init__.py +++ b/python/sglang/kernels/ops/elementwise/__init__.py @@ -13,6 +13,18 @@ from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec __all__ = [] +for _fn in ("hc_combine_gate", "hc_combine_apply", "hc_combine_apply_norm"): + register_kernel( + KernelSpec( + op=f"elementwise.{_fn}", + backend=KernelBackend.JIT, + target=f"sglang.kernels.ops.elementwise.hc_combine_decode:{_fn}", + capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}), + description="BF16 HC decode with four branches of width 2560 and PDL.", + ) + ) +del _fn + # Public entry points inventoried by logical operator group (RFC #29630). register_kernel( diff --git a/python/sglang/kernels/ops/elementwise/elementwise.py b/python/sglang/kernels/ops/elementwise/elementwise.py index 9fffc789ff..b02751674e 100644 --- a/python/sglang/kernels/ops/elementwise/elementwise.py +++ b/python/sglang/kernels/ops/elementwise/elementwise.py @@ -488,6 +488,7 @@ def _fused_gate_sigmoid_mul_add_kernel( BLOCK_SIZE: tl.constexpr, DO_ADD: tl.constexpr = True, USE_PDL: tl.constexpr = False, + GATE_ONLY: tl.constexpr = False, ): pid = tl.program_id(axis=0).to(tl.int64) row_offset = pid * hidden_dim @@ -503,23 +504,30 @@ def _fused_gate_sigmoid_mul_add_kernel( h = tl.load(hidden_states_ptr + row_offset + offsets, mask=mask, other=0.0).to( tl.float32 ) - s = tl.load(shared_output_ptr + row_offset + offsets, mask=mask, other=0.0).to( - tl.float32 - ) - if DO_ADD: - f = tl.load(output_ptr + row_offset + offsets, mask=mask, other=0.0).to( + if not GATE_ONLY: + s = tl.load(shared_output_ptr + row_offset + offsets, mask=mask, other=0.0).to( tl.float32 ) + if DO_ADD: + f = tl.load(output_ptr + row_offset + offsets, mask=mask, other=0.0).to( + tl.float32 + ) - if USE_PDL: - tl.extra.cuda.gdc_launch_dependents() + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() gate_val = tl.sigmoid(tl.sum(h * w, axis=0)) - result = gate_val * s - if DO_ADD: - result += f - - tl.store(output_ptr + row_offset + offsets, result, mask=mask) + if GATE_ONLY: + # Preserve FP32 for the later shared-add FMA, without an intermediate + # BF16 gated shared output. Keep the standalone gate's PDL ordering. + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + tl.store(output_ptr + pid, gate_val) + else: + result = gate_val * s + if DO_ADD: + result += f + tl.store(output_ptr + row_offset + offsets, result, mask=mask) def _launch_fused_gate_sigmoid_mul( diff --git a/python/sglang/kernels/ops/elementwise/hc_combine_decode.py b/python/sglang/kernels/ops/elementwise/hc_combine_decode.py new file mode 100644 index 0000000000..45261bc27f --- /dev/null +++ b/python/sglang/kernels/ops/elementwise/hc_combine_decode.py @@ -0,0 +1,53 @@ +"""Separate HC gate preparation and combine, with optional next-branch norm. + +The decode specialization accepts four branches of 2560 BF16 values. Gate +partials belong to each invocation, so overlapping sublayers never reuse them. +""" + +import torch + +from sglang.kernels.jit.utils import cache_once, load_jit + + +@cache_once +def _jit_module(): + return load_jit( + "hc_combine_decode", + cuda_files=["elementwise/hc_combine_decode.cuh"], + cuda_wrappers=[ + ("gate", "HcCombineDecode::gate"), + ("apply", "HcCombineDecode::apply"), + ("apply_norm", "HcCombineNormDecode<2>::run"), + ], + ) + + +def hc_combine_gate(normed: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + partials = torch.empty( + (normed.shape[0], 8, 4), device=normed.device, dtype=torch.float32 + ) + _jit_module().gate(normed, weight, partials) + return partials + + +def hc_combine_apply( + block: torch.Tensor, residual: torch.Tensor, partials: torch.Tensor +) -> torch.Tensor: + combined = torch.empty_like(residual) + _jit_module().apply(block, residual, partials, combined) + return combined + + +def hc_combine_apply_norm( + block: torch.Tensor, + residual: torch.Tensor, + partials: torch.Tensor, + weight: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor]: + combined = torch.empty_like(residual) + normalized = torch.empty_like(residual) + _jit_module().apply_norm( + block, residual, partials, weight, combined, normalized, eps + ) + return combined, normalized diff --git a/python/sglang/kernels/ops/moe/__init__.py b/python/sglang/kernels/ops/moe/__init__.py index 9ae48045b0..1f75216749 100644 --- a/python/sglang/kernels/ops/moe/__init__.py +++ b/python/sglang/kernels/ops/moe/__init__.py @@ -19,6 +19,16 @@ if TYPE_CHECKING: _CUDA = frozenset({CapabilityRequirement.CUDA}) _HIP = frozenset({CapabilityRequirement.HIP}) +register_kernel( + KernelSpec( + op="moe.shared_expert_gate", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.moe.shared_expert_gate:shared_expert_gate", + capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}), + description="FP32 sigmoid gate from a single BF16 row of width 2560, with PDL.", + ) +) + register_kernel( KernelSpec( op="moe.moe_align_block_size", diff --git a/python/sglang/kernels/ops/moe/shared_expert_gate.py b/python/sglang/kernels/ops/moe/shared_expert_gate.py new file mode 100644 index 0000000000..52283dbb20 --- /dev/null +++ b/python/sglang/kernels/ops/moe/shared_expert_gate.py @@ -0,0 +1,35 @@ +"""FP32 sigmoid gate, preserving the shared-expert fused-add reduction.""" + +import torch + +from sglang.kernels.ops.elementwise.elementwise import ( + _fused_gate_sigmoid_mul_add_kernel, +) + + +def shared_expert_gate( + hidden: torch.Tensor, weight: torch.Tensor, out: torch.Tensor | None = None +) -> torch.Tensor: + assert hidden.is_cuda and weight.device == hidden.device + assert hidden.shape == (1, 2560) and hidden.is_contiguous() + assert hidden.dtype == weight.dtype == torch.bfloat16 + assert weight.numel() == 2560 and weight.is_contiguous() + gate = out + if gate is None: + gate = torch.empty(hidden.shape[0], device=hidden.device, dtype=torch.float32) + assert gate.shape == (hidden.shape[0],) and gate.dtype == torch.float32 + assert gate.device == hidden.device and gate.is_contiguous() + _fused_gate_sigmoid_mul_add_kernel[(hidden.shape[0],)]( + hidden, + weight, + None, + gate, + hidden_dim=2560, + BLOCK_SIZE=4096, + DO_ADD=False, + USE_PDL=True, + GATE_ONLY=True, + num_warps=16, + launch_pdl=True, + ) + return gate diff --git a/python/sglang/srt/layers/hyperconnection.py b/python/sglang/srt/layers/hyperconnection.py index 51671019d5..7329cb5740 100644 --- a/python/sglang/srt/layers/hyperconnection.py +++ b/python/sglang/srt/layers/hyperconnection.py @@ -1,3 +1,4 @@ +from functools import lru_cache from typing import Optional import msgspec @@ -8,6 +9,11 @@ import torch.nn.functional as F from sglang.kernels.ops.gemm.hc_mix import fused_hc_mix, fused_hc_mix_supported +@lru_cache(None) +def _hc_gate_stream(device: int) -> torch.cuda.Stream: + return torch.cuda.Stream(device=device) + + class HyperConnectionConfig(msgspec.Struct, frozen=True): hc_count: int = 4 hidden_size: int = 64 @@ -159,6 +165,7 @@ class GatedResidual(HyperConnectionBase): ) self._mix_up_weight_padded = None + self._gate_stream = None if use_combine: self.block_inject_weight = nn.Linear( self.hidden_size * self.hc_count, @@ -177,6 +184,16 @@ class GatedResidual(HyperConnectionBase): and vecs % (8 * 160) == 0 and (self.hidden_size // 8) % (vecs // 8) == 0 ) + if ( + self.hc_count == 4 + and self.hidden_size == 2560 + and config.hc_per_branch_norm + and config.params_dtype == torch.bfloat16 + and torch.cuda.is_available() + and torch.cuda.get_device_capability()[0] == 10 + ): + # Create before graph capture; all layers on a device share it. + self._gate_stream = _hc_gate_stream(torch.cuda.current_device()) def _mix_compute( hyper_input_normed: torch.Tensor, @@ -216,15 +233,23 @@ class GatedResidual(HyperConnectionBase): self._mix_compute = torch.compile(_mix_compute) self._combine_compute = torch.compile(_combine_compute) - def mix(self, hyper_input: torch.Tensor): + def mix( + self, + hyper_input: torch.Tensor, + normalized_input: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]]: assert hyper_input.shape[-1] == self.hc_count * self.hidden_size if hyper_input.shape[0] == 0: mixed_input = hyper_input.new_empty( (*hyper_input.shape[:-1], self.hidden_size), dtype=self.params_dtype ) - return mixed_input, (hyper_input, hyper_input) + return mixed_input, (hyper_input, hyper_input, None) - if self.config.hc_per_branch_norm: + if normalized_input is not None: + assert self.config.hc_per_branch_norm + assert normalized_input.shape == hyper_input.shape + hyper_input_normed = normalized_input + elif self.config.hc_per_branch_norm: hyper_input_normed = self.hc_norm(hyper_input) else: hyper_input_normed = self.hc_norm( @@ -272,10 +297,74 @@ class GatedResidual(HyperConnectionBase): self.hc_count, self.hidden_size, ).to(self.params_dtype) - return mixed_input, (hyper_input, hyper_input_normed) + partials = None + stream = self._gate_stream + if ( + stream is not None + and hyper_input.shape[0] == 1 + and hyper_input.dtype == torch.bfloat16 + and not torch.compiler.is_compiling() + ): + from sglang.kernels.ops.elementwise.hc_combine_decode import hc_combine_gate - def combine(self, block_output: torch.Tensor, residuals) -> torch.Tensor: - hyper_input, hyper_input_normed = residuals + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + partials = hc_combine_gate( + hyper_input_normed, self.block_inject_weight.weight + ) + hyper_input_normed.record_stream(stream) + return mixed_input, (hyper_input, hyper_input_normed, partials) + + def combine_and_normalize( + self, + block_output: torch.Tensor, + residuals: tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]], + next_norm: Optional[GroupedGemmaRMSNorm], + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + """Return the residual and its next HC normalization explicitly. + + The BF16 combine result is retained before calculating the next norm. + Callers must only pass the norm when no PLE or other update intervenes. + """ + hyper_input, _, partials = residuals + if ( + partials is not None + and next_norm is not None + and next_norm.group_size == self.hidden_size + and next_norm.weight.dtype == torch.bfloat16 + and block_output.dtype == torch.bfloat16 + ): + from sglang.kernels.ops.elementwise.hc_combine_decode import ( + hc_combine_apply_norm, + ) + + current = torch.cuda.current_stream() + current.wait_stream(self._gate_stream) + partials.record_stream(current) + return hc_combine_apply_norm( + block_output, + hyper_input, + partials, + next_norm.weight, + next_norm.variance_epsilon, + ) + return self.combine(block_output, residuals), None + + def combine( + self, + block_output: torch.Tensor, + residuals: tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]], + ) -> torch.Tensor: + hyper_input, hyper_input_normed, partials = residuals + if partials is not None: + from sglang.kernels.ops.elementwise.hc_combine_decode import ( + hc_combine_apply, + ) + + current = torch.cuda.current_stream() + current.wait_stream(self._gate_stream) + partials.record_stream(current) + return hc_combine_apply(block_output, hyper_input, partials) assert hyper_input.shape[-1] == self.hc_count * self.hidden_size assert block_output.shape[-1] == self.hidden_size if block_output.shape[0] == 0: diff --git a/python/sglang/srt/layers/layer_boundary/exit.py b/python/sglang/srt/layers/layer_boundary/exit.py index e0d7fcffe2..94d0dc6e3b 100644 --- a/python/sglang/srt/layers/layer_boundary/exit.py +++ b/python/sglang/srt/layers/layer_boundary/exit.py @@ -100,10 +100,14 @@ class ExitPolicy: hidden_states: torch.Tensor, stream: ResidualStream, forward_batch: ForwardBatch, + *, + already_reduced: bool = False, ): """Complete the output of the operation-scheduled producer path, which never defers: the sum it owes, then its move and write-back.""" steps = self.plan.path_for(forward_batch) + if already_reduced and steps.output_move_completes_sum: + raise ValueError("already-reduced output cannot use a reduce-scatter move") hidden_states, residual = self._complete_now( hidden_states, stream.residual, @@ -111,7 +115,8 @@ class ExitPolicy: dp_step=None, steps=steps, output_move=steps.output_move, - owes_sum=steps.output.group is not None + owes_sum=not already_reduced + and steps.output.group is not None and not steps.output_move_completes_sum, ) return self._record_output( diff --git a/python/sglang/srt/layers/layer_boundary/residual/gated.py b/python/sglang/srt/layers/layer_boundary/residual/gated.py index 6b143cf66e..4feef15630 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/gated.py +++ b/python/sglang/srt/layers/layer_boundary/residual/gated.py @@ -29,11 +29,11 @@ class GatedResidualState: per-stream gates, and writes its output back scaled by an injection coefficient computed from that same normalized residual. - Unlike the hyper-connection streams in mhc.py and ihc.py, whose read - produces the coefficient its write-back consumes, the coefficient here is - computed at write time. What has to survive from a stage's read to its - write-back is therefore the normalized residual itself, which this state - holds. Parameters belong to the owning layer's modules. + The read carries its normalized residual to write-back. Decode may also + prepare gate partials during the read; otherwise write-back computes the + coefficient from that normalized residual. Both follow the residual rows + through any slicing and are cleared after the FFN write-back. + Parameters belong to the owning layer's modules. Args: expand: Widens the layer stack's input into the stream representation, @@ -56,6 +56,7 @@ class GatedResidualState: attn_reads_every_row: bool = False # Produced by a stage's read and consumed by that same stage's write-back. normed: Optional[torch.Tensor] = None + gate_partials: Optional[torch.Tensor] = None def _read(self, mix, residual, out_norm): if out_norm is not None: @@ -63,7 +64,8 @@ class GatedResidualState: "a gated hyper-connection read with a separate norm; the read " "normalizes the streams itself" ) - hidden_states, (residual, self.normed) = mix(residual) + hidden_states, residuals = mix(residual) + residual, self.normed, self.gate_partials = residuals return hidden_states, residual def read_attn_input(self, residual, out_norm=None): @@ -74,18 +76,25 @@ class GatedResidualState: return self._read(self.ffn_mix, residual, out_norm) def apply_attn_combine(self, hidden_states, residual): - return self.attn_combine(hidden_states, (residual, self.normed)) + return self.attn_combine( + hidden_states, (residual, self.normed, self.gate_partials) + ) def apply_ffn_combine(self, hidden_states, residual): - return self.ffn_combine(hidden_states, (residual, self.normed)) + return self.ffn_combine( + hidden_states, (residual, self.normed, self.gate_partials) + ) def clear_coefficients(self): self.normed = None + self.gate_partials = None def slice_residual_attn_tp(self, residual): parallel = get_parallel() rank, size = parallel.attn_tp_rank, parallel.attn_tp_size self.normed = self.normed.tensor_split(size)[rank] + if self.gate_partials is not None: + self.gate_partials = self.gate_partials.tensor_split(size)[rank] return residual.tensor_split(size)[rank] def gather_residual_attn_tp(self, residual): diff --git a/python/sglang/srt/layers/layer_boundary/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py index 66300f895f..0a5bfbdd25 100644 --- a/python/sglang/srt/layers/layer_boundary/stage.py +++ b/python/sglang/srt/layers/layer_boundary/stage.py @@ -262,12 +262,17 @@ class StageBoundary: hidden_states = stream.complete(hidden_states) return hidden_states, stream.snapshot(hidden_states) - def complete_now(self, hidden_states, forward_batch): - """Complete an FFN output outside exit() (the operation-scheduled TBO - path, which never defers): the sum it owes, then the move onto the - rows the layer hands on.""" + def complete_now(self, hidden_states, forward_batch, *, already_reduced=False): + """Complete an FFN output outside exit(), including its residual update. + + already_reduced is for a producer kernel that fused the output sum. + Row movement and residual write-back still belong to this boundary. + """ return self.plan.output.complete_now( - hidden_states, stream_of(forward_batch), forward_batch + hidden_states, + stream_of(forward_batch), + forward_batch, + already_reduced=already_reduced, ) def sum_part(self, hidden_states, forward_batch, group: SumGroup): diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index 5876b7c1d0..2654f7f0f3 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -509,9 +509,16 @@ class FusedMoE(torch.nn.Module): and isinstance(self.quant_method, Fp8MoEMethod) and self.quant_method.block_quant ) + qwen4_bf16_deferred = ( + isinstance(self.quant_method, UnquantizedFusedMoEMethod) + and params_dtype == torch.bfloat16 + and hidden_size == 2560 + and num_experts == 512 + and top_k == 10 + ) self.supports_deferred_finalize = ( get_moe_runner_backend().is_flashinfer_trtllm() - and (nvfp4_deferred or qwen35_fp8_deferred) + and (nvfp4_deferred or qwen35_fp8_deferred or qwen4_bf16_deferred) ) global _deferred_finalize_info_logged if not _deferred_finalize_info_logged: diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py index 4132e33451..3251f680b8 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py @@ -1640,6 +1640,9 @@ def fused_experts_none_to_flashinfer_trtllm_bf16( hidden_states = dispatch_output.hidden_states topk_output = dispatch_output.topk_output + defer_finalize = _deferred_finalize_enabled.get() + if defer_finalize and use_routed_topk: + raise RuntimeError("BF16 deferred finalize requires bypassed TopK") with use_symmetric_memory( get_parallel().tp_group, disabled=not is_allocation_symmetric() @@ -1697,7 +1700,12 @@ def fused_experts_none_to_flashinfer_trtllm_bf16( routed_scaling_factor=runner_config.routed_scaling_factor, tune_max_num_tokens=next_power_of_2(hidden_states.shape[0]), activation_type=activation_type, + do_finalize=not defer_finalize, ) + if defer_finalize: + final_hidden_states = _make_deferred_finalize_output( + final_hidden_states, top_k=topk_config.top_k + ) return StandardCombineInput(hidden_states=final_hidden_states) diff --git a/python/sglang/srt/layers/moe/qwen4_decode.py b/python/sglang/srt/layers/moe/qwen4_decode.py new file mode 100644 index 0000000000..56124d9186 --- /dev/null +++ b/python/sglang/srt/layers/moe/qwen4_decode.py @@ -0,0 +1,95 @@ +"""Single-token BF16 Qwen4 MoE finalize, shared gate and TP4 collective.""" + +from __future__ import annotations + +from functools import lru_cache +from typing import TYPE_CHECKING + +import torch + +from sglang.kernels.ops.communication.all_reduce_fusion import ( + moe_finalize_shared_gate_all_reduce, +) +from sglang.kernels.ops.communication.mp import register_comm_cleanup +from sglang.kernels.ops.moe.shared_expert_gate import shared_expert_gate +from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( + CustomAllReduceV2, +) +from sglang.srt.layers.dp_attention import is_enable_moe_cp_allgather +from sglang.srt.model_executor.runner import get_is_capture_mode +from sglang.srt.runtime_context import get_exec, get_lora, get_parallel, get_spec + +if TYPE_CHECKING: + from sglang.srt.models.qwen2_moe import Qwen2MoeSparseMoeBlock + + +@lru_cache(None) +def _get_decode_comm(tp_comm: CustomAllReduceV2) -> CustomAllReduceV2 | None: + comm = CustomAllReduceV2( + tp_comm.group, + tp_comm.device, + max_pull_size=0, + max_pull_blocks=0, + max_push_size=4 * 1024 * 1024, + max_push_blocks=512, + ) + if comm.disabled: + return None + register_comm_cleanup(comm) + return comm + + +def prepare_qwen4_decode_comm( + mlp: Qwen2MoeSparseMoeBlock, +) -> CustomAllReduceV2 | None: + parallel = get_parallel() + if ( + parallel.tp_size != 4 + or parallel.attn_dp_size != 1 + or parallel.moe_ep_size != 1 + or get_lora().enable_lora + or get_spec().speculative_algorithm is not None + or get_exec().overlap.enable_two_batch_overlap + or is_enable_moe_cp_allgather() + or not mlp.supports_deferred_finalize + or mlp.num_experts != 512 + or mlp.experts.w13_weight.dtype != torch.bfloat16 + or mlp.shared_expert_gate is None + or mlp.enable_shared_expert_fusion + or not torch.cuda.is_available() + or torch.cuda.get_device_capability()[0] != 10 + ): + return None + tp_comm = parallel.tp_group.ca_comm + if not isinstance(tp_comm, CustomAllReduceV2) or tp_comm.disabled: + return None + return _get_decode_comm(tp_comm) + + +def qwen4_decode_moe( + hidden: torch.Tensor, mlp: Qwen2MoeSparseMoeBlock, comm: CustomAllReduceV2 +) -> torch.Tensor: + if mlp.alt_stream is not None and get_is_capture_mode(): + current = torch.cuda.current_stream() + mlp.alt_stream.wait_stream(current) + # FC2 stays on the consuming stream for its PDL dependency. Both + # branches read the input and write independently allocated outputs. + with torch.cuda.stream(mlp.alt_stream): + gate = shared_expert_gate(hidden, mlp.shared_expert_gate.weight) + shared = mlp._forward_shared_experts(hidden, apply_gate=False) + deferred = mlp._forward_router_experts(hidden, defer_finalize=True) + current.wait_stream(mlp.alt_stream) + gate.record_stream(current) + shared.record_stream(current) + else: + gate = shared_expert_gate(hidden, mlp.shared_expert_gate.weight) + shared = mlp._forward_shared_experts(hidden, apply_gate=False) + deferred = mlp._forward_router_experts(hidden, defer_finalize=True) + return moe_finalize_shared_gate_all_reduce( + deferred.gemm2_out, + deferred.expanded_idx_to_permuted_idx, + deferred.expert_weights, + shared, + gate, + comm.obj, + ) diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 7b2069359a..4fd980b651 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -1,7 +1,8 @@ """Inference-only Qwen4-Exp (text + VL) on the Qwen3.5 backbone.""" import math -from contextlib import nullcontext +from contextlib import contextmanager, nullcontext +from dataclasses import dataclass from typing import Any, Iterable, Optional, Set, Tuple, Union import msgspec @@ -34,6 +35,7 @@ from sglang.srt.layers.dp_attention import ( ) from sglang.srt.layers.hyperconnection import ( GatedResidual, + GroupedGemmaRMSNorm, HyperConnectionConfig, ) from sglang.srt.layers.layer_boundary import ( @@ -47,6 +49,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE +from sglang.srt.layers.moe.utils import get_moe_a2a_backend from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.modelopt_quant import ( ModelOptMixedPrecisionConfig, @@ -1422,11 +1425,24 @@ class _MixStreams: """The layer stack's terminal read: the gated mix down to one stream. The stack ends on the mix, with no final norm after it.""" - def __init__(self, hyper_connection_mixer): + def __init__(self, hyper_connection_mixer, normalized_input=None): self.hyper_connection_mixer = hyper_connection_mixer + self.normalized_input = normalized_input def __call__(self, hidden_states): - return self.hyper_connection_mixer.mix(hidden_states)[0] + return self.hyper_connection_mixer.mix( + hidden_states, normalized_input=self.normalized_input + )[0] + + +@dataclass +class _HCNorms: + """Prepared norms for one layer invocation; never carried across PLE or PP.""" + + attn_input: Optional[torch.Tensor] + next_norm: Optional[GroupedGemmaRMSNorm] + ffn_input: Optional[torch.Tensor] = None + output: Optional[torch.Tensor] = None def _has_ple(layer_id, config) -> bool: @@ -1470,6 +1486,7 @@ class Qwen4ExpLayerExtensionMixin: self.ple = None self._ple_forward_batch = None self._ple_batch = None + self._hc_norms = None # The gated reads normalize the streams themselves, so these layers # have neither of the norms their base class builds. @@ -1515,15 +1532,41 @@ class Qwen4ExpLayerExtensionMixin: GatedResidualState( expand=self._widen_streams, attn_mix=self._attn_mix, - ffn_mix=self.mlp_hyper_connection.mix, - attn_combine=self.attn_hyper_connection.combine, - ffn_combine=self.mlp_hyper_connection.combine, + ffn_mix=self._ffn_mix, + attn_combine=self._attn_combine, + ffn_combine=self._ffn_combine, # The PLE embedding is computed for every row. attn_reads_every_row=self.ple is not None, ).residual_ops(), sparse=isinstance(self.mlp, Qwen2MoeSparseMoeBlock), ) + from sglang.srt.layers.moe.qwen4_decode import prepare_qwen4_decode_comm + + self._decode_moe_comm = ( + prepare_qwen4_decode_comm(self.mlp) + if isinstance(self.mlp, Qwen2MoeSparseMoeBlock) + else None + ) + + @contextmanager + def hc_norms(self, normalized_input, next_norm): + norms = _HCNorms(normalized_input, next_norm) + # A prepared norm can only follow its residual without row movement. + self._hc_norms = ( + norms + if get_parallel().attn_dp_size == 1 + and get_parallel().attn_cp_size == 1 + and get_parallel().moe_ep_size == 1 + and not get_attn_tp_context().input_scattered + and get_moe_a2a_backend().is_none() + else None + ) + try: + yield norms + finally: + self._hc_norms = None + def _widen_streams(self, hidden_states: torch.Tensor) -> torch.Tensor: """Open the layer stack's input into one stream per hyper connection. An MTP draft is handed streams that are already widened.""" @@ -1538,7 +1581,42 @@ class Qwen4ExpLayerExtensionMixin: own contribution to the streams and joins them before the read.""" if self.ple is not None: residual = self._add_ple_embedding(residual) - return self.attn_hyper_connection.mix(residual) + normalized_input = ( + self._hc_norms.attn_input + if self._hc_norms is not None and self.ple is None + else None + ) + return self.attn_hyper_connection.mix( + residual, normalized_input=normalized_input + ) + + def _attn_combine(self, hidden_states, residuals): + if self._hc_norms is None: + return self.attn_hyper_connection.combine(hidden_states, residuals) + hidden_states, self._hc_norms.ffn_input = ( + self.attn_hyper_connection.combine_and_normalize( + hidden_states, residuals, self.mlp_hyper_connection.hc_norm + ) + ) + return hidden_states + + def _ffn_mix(self, residual): + normalized_input = ( + self._hc_norms.ffn_input if self._hc_norms is not None else None + ) + return self.mlp_hyper_connection.mix( + residual, normalized_input=normalized_input + ) + + def _ffn_combine(self, hidden_states, residuals): + if self._hc_norms is None: + return self.mlp_hyper_connection.combine(hidden_states, residuals) + hidden_states, self._hc_norms.output = ( + self.mlp_hyper_connection.combine_and_normalize( + hidden_states, residuals, self._hc_norms.next_norm + ) + ) + return hidden_states def _add_ple_embedding(self, residual: torch.Tensor) -> torch.Tensor: forward_batch, ple_batch = self._ple_forward_batch, self._ple_batch @@ -1573,6 +1651,21 @@ class Qwen4ExpLayerExtensionMixin: self, hidden_states: torch.Tensor, forward_batch: ForwardBatch ) -> torch.Tensor: hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + if ( + self._decode_moe_comm is not None + and hidden_states.shape[0] == 1 + and hidden_states.dtype == torch.bfloat16 + and not get_attn_tp_context().input_scattered + and not torch.compiler.is_compiling() + ): + from sglang.srt.layers.moe.qwen4_decode import qwen4_decode_moe + + hidden_states = qwen4_decode_moe( + hidden_states, self.mlp, self._decode_moe_comm + ) + return self.ffn_boundary.complete_now( + hidden_states, forward_batch, already_reduced=True + ) with self.ffn_boundary.exit(forward_batch) as ffn_exit: if isinstance(self.mlp, Qwen2MoeSparseMoeBlock): hidden_states = self.mlp( @@ -1855,8 +1948,6 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): if self.pp_group.is_last_rank else PPMissingLayer() ) - # The stack's terminal read mixes the streams down; there is no final norm. - self._mix_streams = _MixStreams(self.hyper_connection_mixer) def forward( self, @@ -1903,6 +1994,7 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): ngram_eos_token_id=self.ple_ngram_eos_token_id, ) aux_hidden_states = AuxHiddenStateList() + normalized_input = None for i in range(self.start_layer, self.end_layer): layer = self.layers[i] if i + 1 < self.end_layer: @@ -1914,7 +2006,17 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): next_ple.start_prefetch( batch=ple_batch, forward_batch=forward_batch ) - with get_global_expert_distribution_recorder().with_current_layer(i): + next_norm = None + if i + 1 < self.end_layer: + next_layer = self.layers[i + 1] + if next_layer.ple is None: + next_norm = next_layer.attn_hyper_connection.hc_norm + elif self.pp_group.is_last_rank: + next_norm = self.hyper_connection_mixer.hc_norm + with ( + get_global_expert_distribution_recorder().with_current_layer(i), + layer.hc_norms(normalized_input, next_norm) as norms, + ): hidden_states = layer( positions=positions, hidden_states=hidden_states, @@ -1927,6 +2029,8 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): ), ) + normalized_input = norms.output + if breakable_ple: _breakable_commit_ple_batch(batch=ple_batch) else: @@ -1942,7 +2046,9 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): hc_hidden_states = hidden_states hidden_states = residual_batch.final_norm( - hidden_states, forward_batch, self._mix_streams + hidden_states, + forward_batch, + _MixStreams(self.hyper_connection_mixer, normalized_input), ) if not forward_batch.forward_mode.is_idle(): return hidden_states, hc_hidden_states diff --git a/test/manual/layers/test_fused_gate_sigmoid_mul_add.py b/test/manual/layers/test_fused_gate_sigmoid_mul_add.py index f9c204e26e..b7d898be25 100644 --- a/test/manual/layers/test_fused_gate_sigmoid_mul_add.py +++ b/test/manual/layers/test_fused_gate_sigmoid_mul_add.py @@ -3,7 +3,11 @@ import itertools import pytest import torch -from sglang.kernels.ops.elementwise.elementwise import fused_gate_sigmoid_mul_add +from sglang.kernels.ops.elementwise.elementwise import ( + fused_gate_sigmoid_mul, + fused_gate_sigmoid_mul_add, +) +from sglang.kernels.ops.moe.shared_expert_gate import shared_expert_gate from sglang.srt.utils import get_device DTYPES = [torch.float16, torch.bfloat16] @@ -69,6 +73,50 @@ def test_inplace_semantics(): assert fhs.data_ptr() == original_ptr +@pytest.mark.parametrize("weight_scale", [0.0, 0.02, 1.0]) +def test_shared_gate_fp32_and_changed_input_graph_replay(weight_scale): + if torch.version.hip or torch.cuda.get_device_capability()[0] < 9: + pytest.skip("The standalone shared gate uses CUDA PDL (SM90+).") + hidden = torch.randn(1, 2560, device=DEVICE, dtype=torch.bfloat16) + weight = torch.randn(2560, device=DEVICE, dtype=torch.bfloat16) * weight_scale + shared = torch.randn_like(hidden) + routed = torch.randn_like(hidden) + gate = torch.empty(1, device=DEVICE, dtype=torch.float32) + combined = torch.empty_like(hidden) + + def run(): + shared_expert_gate(hidden, weight, out=gate) + gated = fused_gate_sigmoid_mul(hidden, weight, shared) + combined.copy_(routed) + fused_gate_sigmoid_mul_add(hidden, weight, shared, combined) + return gated + + run() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + gated = run() + for _ in range(3): + hidden.normal_() + shared.normal_() + routed.normal_() + graph.replay() + expected_gate = torch.sigmoid((hidden.float() * weight.float()).sum(-1)) + torch.testing.assert_close(gate, expected_gate, rtol=1e-5, atol=1e-6) + # Both full-output variants consume the same FP32 gate without + # rounding the gated shared branch before the final add. + torch.testing.assert_close( + gated, (gate[:, None] * shared.float()).to(hidden.dtype), rtol=0, atol=0 + ) + torch.testing.assert_close( + combined, + torch.addcmul(routed.float(), gate[:, None], shared.float()).to( + hidden.dtype + ), + rtol=0, + atol=0, + ) + + if __name__ == "__main__": import sys diff --git a/test/registered/unit/kernels/test_kernels_namespace.py b/test/registered/unit/kernels/test_kernels_namespace.py index bd3915465b..5de1f6a5f7 100644 --- a/test/registered/unit/kernels/test_kernels_namespace.py +++ b/test/registered/unit/kernels/test_kernels_namespace.py @@ -255,6 +255,7 @@ def test_reclassified_public_entry_points_are_inventoried(): "embeddings.qwen4_ngram", "elementwise.qwen4_gate", "elementwise.row_scale", + "elementwise.hc_combine_decode", "mamba.qwen4_short_conv", "mamba.lfm_short_conv", "gemm.dsv4_wo_a", @@ -268,6 +269,7 @@ def test_reclassified_public_entry_points_are_inventoried(): "layernorm.grouped_gemma_rmsnorm", "moe.dsv4", "moe.gemma4_routing", + "moe.shared_expert_gate", "memory.adler32", "memory.row_compact", "mm.process.image", @@ -293,6 +295,10 @@ def test_reclassified_public_entry_points_are_inventoried(): [ ("gemm.hopper_bf16_gemv", (9, 0), True), ("gemm.hopper_bf16_gemv", (10, 0), False), + ("elementwise.hc_combine_apply_norm", (8, 9), False), + ("elementwise.hc_combine_apply_norm", (9, 0), True), + ("communication.moe_finalize_shared_gate_all_reduce", (8, 9), False), + ("communication.moe_finalize_shared_gate_all_reduce", (10, 3), True), ("gemm.fp8_blockwise_scaled_mm", (12, 0), True), ("gemm.fp8_blockwise_scaled_mm", (12, 1), True), ("gemm.fp8_blockwise_scaled_mm", (10, 0), False), diff --git a/test/registered/unit/layer_boundary/test_gated_residual_ops.py b/test/registered/unit/layer_boundary/test_gated_residual_ops.py index bcef5cb9a6..734292317b 100644 --- a/test/registered/unit/layer_boundary/test_gated_residual_ops.py +++ b/test/registered/unit/layer_boundary/test_gated_residual_ops.py @@ -1,9 +1,15 @@ import unittest +from types import SimpleNamespace from unittest.mock import patch +import test_declared_decoder_boundary as fixture import torch +from sglang.srt.layers.layer_boundary import layer_stack from sglang.srt.layers.layer_boundary.residual.gated import GatedResidualState +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.models.qwen4_exp import _build_qwen4_exp_stages from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -28,17 +34,17 @@ def _normalize(residual, seed): def _mix(seed): - """Stands in for GatedResidual.mix: returns the mixed input and the pair - the write-back consumes, the streams and their normalization.""" + """Stands in for GatedResidual.mix: returns the mixed input and the streams, + their normalization, and optional gate partials consumed at write-back.""" def mix(hyper_input): if hyper_input.shape[0] == 0: empty = hyper_input.new_empty((0, HIDDEN)) # An empty batch puts the un-normalized streams in the second slot. - return empty, (hyper_input, hyper_input) + return empty, (hyper_input, hyper_input, None) normed = _normalize(hyper_input, seed) mixed = normed.unflatten(-1, (HC_COUNT, HIDDEN)).mean(dim=-2) - return mixed, (hyper_input, normed) + return mixed, (hyper_input, normed, None) return mix @@ -46,7 +52,7 @@ def _mix(seed): def _combine(block_output, residuals): """Stands in for GatedResidual.combine: the injection coefficient is computed at write time from the normalized residual the read produced.""" - hyper_input, normed = residuals + hyper_input, normed, _ = residuals if block_output.shape[0] == 0: return hyper_input coefficient = 2 * torch.sigmoid( @@ -99,7 +105,7 @@ class TestGatedResidualOps(CustomTestCase): torch.testing.assert_close( ops.attn_update.update(self.output, residual), - _combine(self.output, (residual, normed)), + _combine(self.output, (residual, normed, None)), ) def test_the_write_back_reads_the_carried_value_not_a_fresh_one(self): @@ -114,7 +120,7 @@ class TestGatedResidualOps(CustomTestCase): state.normed = torch.full_like(state.normed, -3.0) torch.testing.assert_close( ops.attn_update.update(self.output, residual), - _combine(self.output, (residual, state.normed)), + _combine(self.output, (residual, state.normed, None)), ) def test_ffn_write_back_clears_the_carried_value(self): @@ -131,6 +137,68 @@ class TestGatedResidualOps(CustomTestCase): # Nothing survives the layer: the next layer's read produces its own. self.assertIsNone(state.normed) + def test_gate_partials_follow_their_read_and_are_cleared_after_write_back(self): + def mix(seed): + def read(residual): + hidden, (residual, normed, _) = _mix(seed)(residual) + return hidden, (residual, normed, normed.mean(dim=-1, keepdim=True)) + + return read + + def combine(hidden, residuals): + residual, normed, partials = residuals + return _combine(hidden * partials, (residual, normed, None)) + + state = GatedResidualState(_expand, mix(1), mix(2), combine, combine) + ops = state.residual_ops() + residual = _expand(self.hidden) + ops.attn_readout.read(residual, None) + _, attn_residuals = mix(1)(residual) + expected = combine(self.output, attn_residuals) + _, residual = ops.ffn_readout.update_and_read( + ops.attn_update, self.output, residual, None + ) + torch.testing.assert_close(residual, expected) + _, ffn_residuals = mix(2)(expected) + result = ops.ffn_update.update(self.output, residual) + torch.testing.assert_close(result, combine(self.output, ffn_residuals)) + self.assertIsNone(state.normed) + self.assertIsNone(state.gate_partials) + + def test_reduced_ffn_output_still_writes_the_gated_residual_once(self): + for already_reduced in (False, True): + with self.subTest(already_reduced=already_reduced): + state = _state() + residual = _expand(self.hidden) + state.normed = _normalize(residual, 2) + expected = _combine(self.output * 4, (residual, state.normed, None)) + batch = SimpleNamespace( + residual_stream=ResidualStream(residual), + forward_mode=ForwardMode.DECODE, + ) + with ( + fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=4)), + layer_stack(), + ): + _, ffn = _build_qwen4_exp_stages( + state.residual_ops(), + sparse=True, + ) + with patch( + "sglang.srt.layers.layer_boundary.exit.sum_output", + side_effect=lambda value, *args, **kwargs: value * 4, + ) as reduce: + result = ffn.complete_now( + self.output * (4 if already_reduced else 1), + batch, + already_reduced=already_reduced, + ) + self.assertEqual(reduce.call_count, int(not already_reduced)) + torch.testing.assert_close(result, expected) + self.assertIs(batch.residual_stream.residual, result) + self.assertIsNone(batch.residual_stream.pending) + self.assertIsNone(state.normed) + def test_the_ffn_input_is_read_from_the_updated_streams(self): state = _state() ops = state.residual_ops() @@ -141,8 +209,8 @@ class TestGatedResidualOps(CustomTestCase): got_input, got_residual = ops.ffn_readout.update_and_read( ops.attn_update, self.output, residual, None ) - want_residual = _combine(self.output, (residual, attn_normed)) - want_input, (_, want_normed) = _mix(2)(want_residual) + want_residual = _combine(self.output, (residual, attn_normed, None)) + want_input, (_, want_normed, _) = _mix(2)(want_residual) torch.testing.assert_close(got_residual, want_residual) torch.testing.assert_close(got_input, want_input) torch.testing.assert_close(state.normed, want_normed) @@ -161,7 +229,7 @@ class TestGatedResidualOps(CustomTestCase): # The write-back lands on the contributed streams, once. torch.testing.assert_close( ops.attn_update.update(self.output, residual), - _combine(self.output, (entered + contribution, state.normed)), + _combine(self.output, (entered + contribution, state.normed, None)), ) def test_an_empty_batch_keeps_the_streams_and_their_width(self): @@ -236,6 +304,8 @@ class TestGatedResidualOps(CustomTestCase): ops = state.residual_ops() residual = torch.arange(4 * WIDE, dtype=torch.float32).reshape(4, WIDE) state.normed = _normalize(residual, 1) + partials = torch.arange(4 * HC_COUNT).reshape(4, HC_COUNT) + state.gate_partials = partials with patch( "sglang.srt.layers.layer_boundary.residual.gated.get_parallel" ) as parallel: @@ -245,6 +315,7 @@ class TestGatedResidualOps(CustomTestCase): torch.testing.assert_close(sliced, residual[2:]) # The normalized rows must follow the stream rows they scale. torch.testing.assert_close(state.normed, _normalize(residual, 1)[2:]) + torch.testing.assert_close(state.gate_partials, partials[2:]) def test_attn_tp_gather_is_rejected(self): ops = _state().residual_ops()