[Performance] Optimize Qwen3.8-Flash-Next BF16 decode on Blackwell (#42175)
This commit is contained in:
@@ -68,13 +68,14 @@ SGL_DEVICE void barrier_cluster_wait() {
|
||||
|
||||
template <uint32_t kWorldSize, typename WeightT>
|
||||
struct MoeFinalizeAllReduceParams {
|
||||
bf16_t* out; // [num_tokens, kHiddenDim], output-only
|
||||
const bf16_t* gemm2; // [P, kHiddenDim], permuted / padded rows
|
||||
const int32_t* idx; // [num_tokens * kTopK], -1 = dropped slot
|
||||
const WeightT* weights; // [num_tokens, kTopK], scaling already folded in
|
||||
const bf16_t* shared; // [num_tokens, kHiddenDim] (kHasShared only)
|
||||
const bf16_t* norm_weight; // [kHiddenDim] (kNorm only)
|
||||
float norm_eps; // kNorm only
|
||||
bf16_t* out; // [num_tokens, kHiddenDim], output-only
|
||||
const bf16_t* gemm2; // [P, kHiddenDim], permuted / padded rows
|
||||
const int32_t* idx; // [num_tokens * kTopK], -1 = dropped slot
|
||||
const WeightT* weights; // [num_tokens, kTopK], scaling already folded in
|
||||
const bf16_t* shared; // [num_tokens, kHiddenDim] (kHasShared only)
|
||||
const float* shared_gate = nullptr; // Optional FP32 sigmoid, one per token.
|
||||
const bf16_t* norm_weight; // [kHiddenDim] (kNorm only)
|
||||
float norm_eps; // kNorm only
|
||||
// Caller's promise that everything read before the PDL wait is complete when
|
||||
// the predecessor merely *triggers*: no all-reduce on this plane right before
|
||||
// it, and the routing metadata's producers finished (PDL completion is not
|
||||
@@ -199,7 +200,14 @@ struct RowClusterTrait {
|
||||
// --- stage 1: the deferred finalize of one 16B vector ------------------------
|
||||
// The shared-expert vector is loaded first so that load is in flight while the
|
||||
// routing rows and the kTopK gathers are fetched.
|
||||
template <uint32_t kHiddenDim, uint32_t kTopK, bool kHasShared, bool kUsePDL, uint32_t kWorldSize, typename WeightT>
|
||||
template <
|
||||
uint32_t kHiddenDim,
|
||||
uint32_t kTopK,
|
||||
bool kHasShared,
|
||||
bool kUsePDL,
|
||||
uint32_t kWorldSize,
|
||||
typename WeightT,
|
||||
bool kGatedShared = false>
|
||||
SGL_DEVICE StageVec
|
||||
finalize_vec(const MoeFinalizeAllReduceParams<kWorldSize, WeightT>& params, uint32_t token, uint32_t hvec) {
|
||||
using namespace device;
|
||||
@@ -263,7 +271,12 @@ finalize_vec(const MoeFinalizeAllReduceParams<kWorldSize, WeightT>& params, uint
|
||||
if constexpr (kHasShared) {
|
||||
const auto routed = cast<fp32x2_t>(cast<bf16x2_t>(acc[j]));
|
||||
const auto sh = cast<fp32x2_t>(shared_in[j]);
|
||||
out[j] = cast<bf16x2_t>(fp32x2_t{routed.x + sh.x, routed.y + sh.y});
|
||||
if constexpr (kGatedShared) {
|
||||
const float gate = params.shared_gate[token];
|
||||
out[j] = cast<bf16x2_t>(fp32x2_t{fmaf(gate, sh.x, routed.x), fmaf(gate, sh.y, routed.y)});
|
||||
} else {
|
||||
out[j] = cast<bf16x2_t>(fp32x2_t{routed.x + sh.x, routed.y + sh.y});
|
||||
}
|
||||
} else {
|
||||
out[j] = cast<bf16x2_t>(acc[j]);
|
||||
}
|
||||
@@ -287,7 +300,8 @@ template <
|
||||
typename WeightT,
|
||||
bool kMhc = false,
|
||||
bool kQuant = false,
|
||||
bool kCollapse = false>
|
||||
bool kCollapse = false,
|
||||
bool kGatedShared = false>
|
||||
__global__ __launch_bounds__(RowClusterTrait<kHiddenDim, kClusterSize>::kBlockSize)
|
||||
__cluster_dims__(1, kClusterSize, 1) void moe_finalize_all_reduce_kernel(
|
||||
const __grid_constant__ MoeFinalizeAllReduceParams<kWorldSize, WeightT> params) {
|
||||
@@ -342,7 +356,8 @@ __global__ __launch_bounds__(RowClusterTrait<kHiddenDim, kClusterSize>::kBlockSi
|
||||
// stage 1: finalize this row's vector in registers and push it to every peer
|
||||
const auto vid = row_idx * kRowVecs + hvec;
|
||||
{
|
||||
auto vec = finalize_vec<kHiddenDim, kTopK, kHasShared, kUsePDL>(params, row_idx, hvec);
|
||||
auto vec =
|
||||
finalize_vec<kHiddenDim, kTopK, kHasShared, kUsePDL, kWorldSize, WeightT, kGatedShared>(params, row_idx, hvec);
|
||||
Lamport::clear_pos_zero(vec.data());
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kWorldSize; ++i) {
|
||||
@@ -456,7 +471,8 @@ template <
|
||||
typename WeightT,
|
||||
bool kMhc = false,
|
||||
bool kQuant = false,
|
||||
bool kCollapse = false>
|
||||
bool kCollapse = false,
|
||||
bool kGatedShared = false>
|
||||
struct MoeFinalizeAllReduceKernel {
|
||||
private:
|
||||
static_assert(std::is_same_v<WeightT, bf16_t> || std::is_same_v<WeightT, fp32_t>);
|
||||
@@ -476,7 +492,8 @@ struct MoeFinalizeAllReduceKernel {
|
||||
WeightT,
|
||||
kMhc,
|
||||
kQuant,
|
||||
kCollapse>;
|
||||
kCollapse,
|
||||
kGatedShared>;
|
||||
|
||||
public:
|
||||
/// out = [allreduce over ranks of] finalize(gemm2_out, idx, weights) [+ shared] [-> RMSNorm(norm_weight, eps)].
|
||||
@@ -509,6 +526,37 @@ struct MoeFinalizeAllReduceKernel {
|
||||
std::nullopt);
|
||||
}
|
||||
|
||||
// Preserve routed BF16 rounding, then apply the FP32 shared gate with FMA.
|
||||
static void run_shared_gate(
|
||||
CommunicatorRef ref,
|
||||
TensorView out,
|
||||
TensorView gemm2_out,
|
||||
TensorView permuted_idx,
|
||||
TensorView expert_weights,
|
||||
TensorView shared_output,
|
||||
TensorView shared_gate) {
|
||||
static_assert(kGatedShared && !kMhc);
|
||||
run_impl(
|
||||
ref,
|
||||
out,
|
||||
gemm2_out,
|
||||
permuted_idx,
|
||||
expert_weights,
|
||||
shared_output,
|
||||
std::nullopt,
|
||||
0.0,
|
||||
false,
|
||||
std::nullopt,
|
||||
std::nullopt,
|
||||
std::nullopt,
|
||||
std::nullopt,
|
||||
std::nullopt,
|
||||
std::nullopt,
|
||||
std::nullopt,
|
||||
std::nullopt,
|
||||
shared_gate);
|
||||
}
|
||||
|
||||
/// Finalize + all-reduce + HC=4 post; original reduced output is retained.
|
||||
static void run_mhc_post(
|
||||
CommunicatorRef ref,
|
||||
@@ -628,7 +676,8 @@ struct MoeFinalizeAllReduceKernel {
|
||||
std::optional<TensorView> pre = std::nullopt,
|
||||
std::optional<TensorView> normalized = std::nullopt,
|
||||
std::optional<TensorView> quantized = std::nullopt,
|
||||
std::optional<TensorView> scales = std::nullopt) {
|
||||
std::optional<TensorView> scales = std::nullopt,
|
||||
std::optional<TensorView> shared_gate = std::nullopt) {
|
||||
using namespace host;
|
||||
const auto& comm = *ref.get();
|
||||
const auto& push = comm.get_push_obj();
|
||||
@@ -664,6 +713,10 @@ struct MoeFinalizeAllReduceKernel {
|
||||
.with_device<kDLCUDA>(device)
|
||||
.verify(shared_output.value());
|
||||
}
|
||||
if constexpr (kGatedShared) {
|
||||
CHECK_HOST(shared_output.has_value() && shared_gate.has_value());
|
||||
TensorMatcher({T}).with_dtype<fp32_t>().with_device<kDLCUDA>(device).verify(shared_gate.value());
|
||||
}
|
||||
if (norm_weight.has_value()) {
|
||||
TensorMatcher({kHiddenDim})
|
||||
.with_strides({1})
|
||||
@@ -717,6 +770,7 @@ struct MoeFinalizeAllReduceKernel {
|
||||
.idx = static_cast<const int32_t*>(permuted_idx.data_ptr()),
|
||||
.weights = static_cast<const WeightT*>(expert_weights.data_ptr()),
|
||||
.shared = shared_output.has_value() ? static_cast<const bf16_t*>(shared_output.value().data_ptr()) : nullptr,
|
||||
.shared_gate = shared_gate.has_value() ? static_cast<const float*>(shared_gate.value().data_ptr()) : nullptr,
|
||||
.norm_weight = norm_weight.has_value() ? static_cast<const bf16_t*>(norm_weight.value().data_ptr()) : nullptr,
|
||||
.norm_eps = static_cast<float>(eps),
|
||||
.prefetch_metadata = prefetch_metadata,
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
#pragma once
|
||||
#include "hc_combine.cuh"
|
||||
|
||||
namespace sglang {
|
||||
// Expose the existing two stock stages separately; all arithmetic is unchanged.
|
||||
struct HcCombineDecode {
|
||||
static void gate(tvm::ffi::TensorView normed, tvm::ffi::TensorView weight, tvm::ffi::TensorView partials) {
|
||||
using namespace host;
|
||||
SymbolicSize m{"rows"};
|
||||
SymbolicDevice dev;
|
||||
dev.set_options<kDLCUDA>();
|
||||
TensorMatcher({m, 10240}).with_dtype<bf16_t>().with_device(dev).verify(normed);
|
||||
TensorMatcher({4, 10240}).with_dtype<bf16_t>().with_device(dev).verify(weight);
|
||||
TensorMatcher({m, 8, 4}).with_dtype<fp32_t>().with_device(dev).verify(partials);
|
||||
const HcCombineSplitParams params{
|
||||
.block_output = nullptr,
|
||||
.residual = nullptr,
|
||||
.normed_residual = normed.data_ptr(),
|
||||
.inject_weight = weight.data_ptr(),
|
||||
.output = nullptr,
|
||||
.partials = static_cast<float*>(partials.data_ptr())};
|
||||
LaunchKernel(dim3(m.unwrap(), 32, 1), 32, dev.unwrap())
|
||||
.enable_pdl(true)(hc_combine_gate_kernel<4, 2560, true, bf16_t>, params);
|
||||
}
|
||||
static void apply(
|
||||
tvm::ffi::TensorView block,
|
||||
tvm::ffi::TensorView residual,
|
||||
tvm::ffi::TensorView partials,
|
||||
tvm::ffi::TensorView out) {
|
||||
using namespace host;
|
||||
SymbolicSize m{"rows"};
|
||||
SymbolicDevice dev;
|
||||
dev.set_options<kDLCUDA>();
|
||||
TensorMatcher({m, 2560}).with_dtype<bf16_t>().with_device(dev).verify(block);
|
||||
TensorMatcher({m, 10240}).with_dtype<bf16_t>().with_device(dev).verify(residual).verify(out);
|
||||
TensorMatcher({m, 8, 4}).with_dtype<fp32_t>().with_device(dev).verify(partials);
|
||||
const HcCombineSplitParams params{
|
||||
.block_output = block.data_ptr(),
|
||||
.residual = residual.data_ptr(),
|
||||
.normed_residual = nullptr,
|
||||
.inject_weight = nullptr,
|
||||
.output = out.data_ptr(),
|
||||
.partials = static_cast<float*>(partials.data_ptr())};
|
||||
LaunchKernel(dim3(m.unwrap(), 8, 1), 160, dev.unwrap())
|
||||
.enable_pdl(true)(hc_combine_apply_kernel<4, 2560, true, bf16_t>, params);
|
||||
}
|
||||
};
|
||||
struct HcCombineNormDecodeParams {
|
||||
const bf16_t* block;
|
||||
const bf16_t* residual;
|
||||
const float* partials;
|
||||
const bf16_t* weight;
|
||||
bf16_t* combined;
|
||||
bf16_t* normalized;
|
||||
float eps;
|
||||
};
|
||||
|
||||
template <int kParts>
|
||||
__global__
|
||||
__launch_bounds__(160) void hc_combine_norm_decode_kernel(const HcCombineNormDecodeParams __grid_constant__ params) {
|
||||
using namespace device;
|
||||
using BFloat16x2 = packed_t<bf16_t>;
|
||||
using Storage = AlignedVector<BFloat16x2, 8>;
|
||||
constexpr uint32_t kThreads = 160;
|
||||
constexpr uint32_t kWarps = 5;
|
||||
static_assert(kThreads % kParts == 0);
|
||||
const uint32_t row = blockIdx.x / (4 * kParts);
|
||||
const uint32_t branch = (blockIdx.x / kParts) % 4;
|
||||
const uint32_t part = blockIdx.x % kParts;
|
||||
const uint32_t tid = threadIdx.x;
|
||||
const auto gmem = tile::Memory<Storage>::cta(kThreads);
|
||||
const auto residual = params.residual + (row * 4 + branch) * 2560;
|
||||
const auto block_output = params.block + row * 2560;
|
||||
const auto norm_weight = params.weight + branch * 2560;
|
||||
__shared__ float warp_sums[kWarpThreads];
|
||||
PDLWaitPrimary<true>();
|
||||
float total = 0.0f;
|
||||
#pragma unroll
|
||||
for (uint32_t split = 0; split < 8; ++split)
|
||||
total += params.partials[(row * 8 + split) * 4 + branch];
|
||||
const float gate = 2.0f / (1.0f + math::exp(-total / 4));
|
||||
const auto residual_values = gmem.load(residual, 0);
|
||||
const auto block_values = gmem.load(block_output, 0);
|
||||
const auto weight_values = gmem.load(norm_weight, 0);
|
||||
Storage combined;
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < 8; ++i) {
|
||||
const auto [rx, ry] = cast<fp32x2_t>(residual_values[i]);
|
||||
const auto [yx, yy] = cast<fp32x2_t>(block_values[i]);
|
||||
// Match the stock apply's FMA and its BF16 store before normalization.
|
||||
combined[i] = cast<BFloat16x2>(fp32x2_t{rx + gate * yx, ry + gate * yy});
|
||||
}
|
||||
float sum = 0.0f;
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < 8; ++i) {
|
||||
const auto [x, y] = cast<fp32x2_t>(combined[i]);
|
||||
sum += x * x + y * y;
|
||||
}
|
||||
sum = warp::reduce_sum(sum);
|
||||
const auto warp_id = tid / kWarpThreads;
|
||||
warp_sums[warp_id] = sum;
|
||||
__syncthreads();
|
||||
if (warp_id == 0) {
|
||||
const auto local_sum = tid < kWarps ? warp_sums[tid] : 0.0f;
|
||||
sum = warp::reduce_sum(local_sum);
|
||||
warp_sums[tid] = math::rsqrt(sum / 2560 + params.eps);
|
||||
}
|
||||
__syncthreads();
|
||||
const float inv_rms = warp_sums[warp_id];
|
||||
if (tid >= part * (kThreads / kParts) && tid < (part + 1) * (kThreads / kParts)) {
|
||||
Storage normalized;
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < 8; ++i) {
|
||||
const auto [x, y] = cast<fp32x2_t>(combined[i]);
|
||||
const auto [wx, wy] = cast<fp32x2_t>(weight_values[i]);
|
||||
normalized[i] = cast<BFloat16x2>(fp32x2_t{x * inv_rms * (1.0f + wx), y * inv_rms * (1.0f + wy)});
|
||||
}
|
||||
gmem.store(params.combined + (row * 4 + branch) * 2560, combined, 0);
|
||||
gmem.store(params.normalized + (row * 4 + branch) * 2560, normalized, 0);
|
||||
}
|
||||
PDLTriggerSecondary<true>();
|
||||
}
|
||||
|
||||
template <int kParts>
|
||||
struct HcCombineNormDecode {
|
||||
static void
|
||||
run(tvm::ffi::TensorView block,
|
||||
tvm::ffi::TensorView residual,
|
||||
tvm::ffi::TensorView partials,
|
||||
tvm::ffi::TensorView weight,
|
||||
tvm::ffi::TensorView combined,
|
||||
tvm::ffi::TensorView normalized,
|
||||
float eps) {
|
||||
using namespace host;
|
||||
SymbolicSize m{"rows"};
|
||||
SymbolicDevice dev;
|
||||
dev.set_options<kDLCUDA>();
|
||||
TensorMatcher({m, 2560}).with_dtype<bf16_t>().with_device(dev).verify(block);
|
||||
TensorMatcher({m, 10240})
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device(dev)
|
||||
.verify(residual)
|
||||
.verify(combined)
|
||||
.verify(normalized);
|
||||
TensorMatcher({10240}).with_dtype<bf16_t>().with_device(dev).verify(weight);
|
||||
TensorMatcher({m, 8, 4}).with_dtype<fp32_t>().with_device(dev).verify(partials);
|
||||
CHECK_HOST(m.unwrap() > 0 && m.unwrap() <= 4);
|
||||
const HcCombineNormDecodeParams p{
|
||||
static_cast<const bf16_t*>(block.data_ptr()),
|
||||
static_cast<const bf16_t*>(residual.data_ptr()),
|
||||
static_cast<const float*>(partials.data_ptr()),
|
||||
static_cast<const bf16_t*>(weight.data_ptr()),
|
||||
static_cast<bf16_t*>(combined.data_ptr()),
|
||||
static_cast<bf16_t*>(normalized.data_ptr()),
|
||||
eps};
|
||||
LaunchKernel(m.unwrap() * 4 * kParts, 160, dev.unwrap()).enable_pdl(true)(hc_combine_norm_decode_kernel<kParts>, p);
|
||||
}
|
||||
};
|
||||
} // namespace sglang
|
||||
@@ -359,7 +359,18 @@ def fused_recurrent_gated_delta_rule_packed_decode(
|
||||
raise ValueError(
|
||||
f"Packed decode kernel only supports NK=1 (got K={K}, BK={BK})."
|
||||
)
|
||||
BV = min(triton.next_power_of_2(V), 32)
|
||||
# Single-token TP4 decode otherwise launches only 48 CTAs and uses over
|
||||
# 200 registers per thread. Smaller value tiles expose 192 independent
|
||||
# CTAs without changing the recurrence or its one-warp K reductions.
|
||||
use_small_value_tile = (
|
||||
B == 1
|
||||
and qkv_dim == 2560
|
||||
and mixed_qkv.dtype == torch.bfloat16
|
||||
and initial_state.dtype == torch.float32
|
||||
and (HV, V, K) == (12, 128, 128)
|
||||
and torch.cuda.get_device_capability(mixed_qkv.device)[0] == 10
|
||||
)
|
||||
BV = min(triton.next_power_of_2(V), 8 if use_small_value_tile else 32)
|
||||
num_stages = 3
|
||||
num_warps = 1
|
||||
|
||||
|
||||
@@ -10,6 +10,16 @@ from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
|
||||
|
||||
__all__ = []
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="communication.moe_finalize_shared_gate_all_reduce",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.communication.all_reduce_fusion:moe_finalize_shared_gate_all_reduce",
|
||||
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}),
|
||||
description="BF16 routed finalize, FP32 shared gate, and clustered all-reduce.",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# Kernels introduced with Kimi-K3, inventoried by logical operator group.
|
||||
for _mod, _fn in [
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Fused deferred-MoE finalize + 1shot push all-reduce [+ RMSNorm] (bf16).
|
||||
|
||||
One entry point, :func:`moe_finalize_all_reduce`, over
|
||||
The base entry point, :func:`moe_finalize_all_reduce`, wraps
|
||||
``csrc/distributed/all_reduce_fusion.cuh``::
|
||||
|
||||
out[t] = allreduce( sum_k expert_weights[t, k] * gemm2_out[idx[t*top_k + k]]
|
||||
@@ -14,8 +14,10 @@ materializes; ``idx == -1`` slots (EP: non-local expert) contribute nothing.
|
||||
Small-batch only: the whole ``[T, hidden]`` bf16 row view must fit one push
|
||||
slot (checked C++-side; :func:`fits_push_slot` lets callers pre-check).
|
||||
|
||||
Needs :func:`register_comm` once per process (the CustomAllReduceV2
|
||||
``Communicator``); the ops key on ``world_size`` alone.
|
||||
The base entry point needs :func:`register_comm` once per process and keys on
|
||||
``world_size`` alone. :func:`moe_finalize_shared_gate_all_reduce` takes the
|
||||
caller's communicator directly and applies an FP32 shared gate after rounding
|
||||
the routed values to BF16, before the collective.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -229,3 +231,70 @@ def moe_finalize_all_reduce(
|
||||
prefetch_metadata,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_shared_gate_module(
|
||||
world_size: int,
|
||||
hidden_dim: int,
|
||||
top_k: int,
|
||||
cluster_size: int,
|
||||
weight_dtype: torch.dtype,
|
||||
) -> Module:
|
||||
require_cluster_launch_arch()
|
||||
assert cluster_size in valid_cluster_sizes(hidden_dim)
|
||||
args = make_cpp_args(
|
||||
world_size,
|
||||
hidden_dim,
|
||||
top_k,
|
||||
cluster_size,
|
||||
is_arch_support_pdl(),
|
||||
weight_dtype,
|
||||
False, # kMhc
|
||||
False, # kQuant
|
||||
False, # kCollapse
|
||||
True, # kGatedShared
|
||||
)
|
||||
return load_jit(
|
||||
"moe_finalize_shared_gate_all_reduce",
|
||||
*args,
|
||||
cuda_files=["distributed/all_reduce_fusion.cuh"],
|
||||
cuda_wrappers=[("run", f"MoeFinalizeAllReduceKernel<{args}>::run_shared_gate")],
|
||||
)
|
||||
|
||||
|
||||
def moe_finalize_shared_gate_all_reduce(
|
||||
gemm2_out: torch.Tensor,
|
||||
expanded_idx_to_permuted_idx: torch.Tensor,
|
||||
expert_weights: torch.Tensor,
|
||||
shared_output: torch.Tensor,
|
||||
shared_gate: torch.Tensor,
|
||||
comm: Communicator,
|
||||
*,
|
||||
cluster_size: Optional[int] = None,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Finalize routed BF16 values, gate the shared branch, then all-reduce.
|
||||
|
||||
``shared_gate`` is FP32 and is applied after the routed BF16 rounding.
|
||||
The caller owns the communicator and joins all input-producing streams.
|
||||
"""
|
||||
_, hidden_dim = shared_output.shape
|
||||
if out is None:
|
||||
out = torch.empty_like(shared_output)
|
||||
_jit_shared_gate_module(
|
||||
comm.world_size,
|
||||
hidden_dim,
|
||||
expert_weights.shape[1],
|
||||
cluster_size or default_cluster_size(hidden_dim),
|
||||
expert_weights.dtype,
|
||||
).run(
|
||||
comm,
|
||||
out,
|
||||
gemm2_out,
|
||||
expanded_idx_to_permuted_idx,
|
||||
expert_weights,
|
||||
shared_output,
|
||||
shared_gate,
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -13,6 +13,18 @@ from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
|
||||
|
||||
__all__ = []
|
||||
|
||||
for _fn in ("hc_combine_gate", "hc_combine_apply", "hc_combine_apply_norm"):
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op=f"elementwise.{_fn}",
|
||||
backend=KernelBackend.JIT,
|
||||
target=f"sglang.kernels.ops.elementwise.hc_combine_decode:{_fn}",
|
||||
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}),
|
||||
description="BF16 HC decode with four branches of width 2560 and PDL.",
|
||||
)
|
||||
)
|
||||
del _fn
|
||||
|
||||
|
||||
# Public entry points inventoried by logical operator group (RFC #29630).
|
||||
register_kernel(
|
||||
|
||||
@@ -488,6 +488,7 @@ def _fused_gate_sigmoid_mul_add_kernel(
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
DO_ADD: tl.constexpr = True,
|
||||
USE_PDL: tl.constexpr = False,
|
||||
GATE_ONLY: tl.constexpr = False,
|
||||
):
|
||||
pid = tl.program_id(axis=0).to(tl.int64)
|
||||
row_offset = pid * hidden_dim
|
||||
@@ -503,23 +504,30 @@ def _fused_gate_sigmoid_mul_add_kernel(
|
||||
h = tl.load(hidden_states_ptr + row_offset + offsets, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
s = tl.load(shared_output_ptr + row_offset + offsets, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
if DO_ADD:
|
||||
f = tl.load(output_ptr + row_offset + offsets, mask=mask, other=0.0).to(
|
||||
if not GATE_ONLY:
|
||||
s = tl.load(shared_output_ptr + row_offset + offsets, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
if DO_ADD:
|
||||
f = tl.load(output_ptr + row_offset + offsets, mask=mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
gate_val = tl.sigmoid(tl.sum(h * w, axis=0))
|
||||
result = gate_val * s
|
||||
if DO_ADD:
|
||||
result += f
|
||||
|
||||
tl.store(output_ptr + row_offset + offsets, result, mask=mask)
|
||||
if GATE_ONLY:
|
||||
# Preserve FP32 for the later shared-add FMA, without an intermediate
|
||||
# BF16 gated shared output. Keep the standalone gate's PDL ordering.
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(output_ptr + pid, gate_val)
|
||||
else:
|
||||
result = gate_val * s
|
||||
if DO_ADD:
|
||||
result += f
|
||||
tl.store(output_ptr + row_offset + offsets, result, mask=mask)
|
||||
|
||||
|
||||
def _launch_fused_gate_sigmoid_mul(
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Separate HC gate preparation and combine, with optional next-branch norm.
|
||||
|
||||
The decode specialization accepts four branches of 2560 BF16 values. Gate
|
||||
partials belong to each invocation, so overlapping sublayers never reuse them.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_module():
|
||||
return load_jit(
|
||||
"hc_combine_decode",
|
||||
cuda_files=["elementwise/hc_combine_decode.cuh"],
|
||||
cuda_wrappers=[
|
||||
("gate", "HcCombineDecode::gate"),
|
||||
("apply", "HcCombineDecode::apply"),
|
||||
("apply_norm", "HcCombineNormDecode<2>::run"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def hc_combine_gate(normed: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
|
||||
partials = torch.empty(
|
||||
(normed.shape[0], 8, 4), device=normed.device, dtype=torch.float32
|
||||
)
|
||||
_jit_module().gate(normed, weight, partials)
|
||||
return partials
|
||||
|
||||
|
||||
def hc_combine_apply(
|
||||
block: torch.Tensor, residual: torch.Tensor, partials: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
combined = torch.empty_like(residual)
|
||||
_jit_module().apply(block, residual, partials, combined)
|
||||
return combined
|
||||
|
||||
|
||||
def hc_combine_apply_norm(
|
||||
block: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
partials: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
combined = torch.empty_like(residual)
|
||||
normalized = torch.empty_like(residual)
|
||||
_jit_module().apply_norm(
|
||||
block, residual, partials, weight, combined, normalized, eps
|
||||
)
|
||||
return combined, normalized
|
||||
@@ -19,6 +19,16 @@ if TYPE_CHECKING:
|
||||
_CUDA = frozenset({CapabilityRequirement.CUDA})
|
||||
_HIP = frozenset({CapabilityRequirement.HIP})
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.shared_expert_gate",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.moe.shared_expert_gate:shared_expert_gate",
|
||||
capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}),
|
||||
description="FP32 sigmoid gate from a single BF16 row of width 2560, with PDL.",
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.moe_align_block_size",
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
"""FP32 sigmoid gate, preserving the shared-expert fused-add reduction."""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.elementwise.elementwise import (
|
||||
_fused_gate_sigmoid_mul_add_kernel,
|
||||
)
|
||||
|
||||
|
||||
def shared_expert_gate(
|
||||
hidden: torch.Tensor, weight: torch.Tensor, out: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
assert hidden.is_cuda and weight.device == hidden.device
|
||||
assert hidden.shape == (1, 2560) and hidden.is_contiguous()
|
||||
assert hidden.dtype == weight.dtype == torch.bfloat16
|
||||
assert weight.numel() == 2560 and weight.is_contiguous()
|
||||
gate = out
|
||||
if gate is None:
|
||||
gate = torch.empty(hidden.shape[0], device=hidden.device, dtype=torch.float32)
|
||||
assert gate.shape == (hidden.shape[0],) and gate.dtype == torch.float32
|
||||
assert gate.device == hidden.device and gate.is_contiguous()
|
||||
_fused_gate_sigmoid_mul_add_kernel[(hidden.shape[0],)](
|
||||
hidden,
|
||||
weight,
|
||||
None,
|
||||
gate,
|
||||
hidden_dim=2560,
|
||||
BLOCK_SIZE=4096,
|
||||
DO_ADD=False,
|
||||
USE_PDL=True,
|
||||
GATE_ONLY=True,
|
||||
num_warps=16,
|
||||
launch_pdl=True,
|
||||
)
|
||||
return gate
|
||||
@@ -1,3 +1,4 @@
|
||||
from functools import lru_cache
|
||||
from typing import Optional
|
||||
|
||||
import msgspec
|
||||
@@ -8,6 +9,11 @@ import torch.nn.functional as F
|
||||
from sglang.kernels.ops.gemm.hc_mix import fused_hc_mix, fused_hc_mix_supported
|
||||
|
||||
|
||||
@lru_cache(None)
|
||||
def _hc_gate_stream(device: int) -> torch.cuda.Stream:
|
||||
return torch.cuda.Stream(device=device)
|
||||
|
||||
|
||||
class HyperConnectionConfig(msgspec.Struct, frozen=True):
|
||||
hc_count: int = 4
|
||||
hidden_size: int = 64
|
||||
@@ -159,6 +165,7 @@ class GatedResidual(HyperConnectionBase):
|
||||
)
|
||||
self._mix_up_weight_padded = None
|
||||
|
||||
self._gate_stream = None
|
||||
if use_combine:
|
||||
self.block_inject_weight = nn.Linear(
|
||||
self.hidden_size * self.hc_count,
|
||||
@@ -177,6 +184,16 @@ class GatedResidual(HyperConnectionBase):
|
||||
and vecs % (8 * 160) == 0
|
||||
and (self.hidden_size // 8) % (vecs // 8) == 0
|
||||
)
|
||||
if (
|
||||
self.hc_count == 4
|
||||
and self.hidden_size == 2560
|
||||
and config.hc_per_branch_norm
|
||||
and config.params_dtype == torch.bfloat16
|
||||
and torch.cuda.is_available()
|
||||
and torch.cuda.get_device_capability()[0] == 10
|
||||
):
|
||||
# Create before graph capture; all layers on a device share it.
|
||||
self._gate_stream = _hc_gate_stream(torch.cuda.current_device())
|
||||
|
||||
def _mix_compute(
|
||||
hyper_input_normed: torch.Tensor,
|
||||
@@ -216,15 +233,23 @@ class GatedResidual(HyperConnectionBase):
|
||||
self._mix_compute = torch.compile(_mix_compute)
|
||||
self._combine_compute = torch.compile(_combine_compute)
|
||||
|
||||
def mix(self, hyper_input: torch.Tensor):
|
||||
def mix(
|
||||
self,
|
||||
hyper_input: torch.Tensor,
|
||||
normalized_input: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]]:
|
||||
assert hyper_input.shape[-1] == self.hc_count * self.hidden_size
|
||||
if hyper_input.shape[0] == 0:
|
||||
mixed_input = hyper_input.new_empty(
|
||||
(*hyper_input.shape[:-1], self.hidden_size), dtype=self.params_dtype
|
||||
)
|
||||
return mixed_input, (hyper_input, hyper_input)
|
||||
return mixed_input, (hyper_input, hyper_input, None)
|
||||
|
||||
if self.config.hc_per_branch_norm:
|
||||
if normalized_input is not None:
|
||||
assert self.config.hc_per_branch_norm
|
||||
assert normalized_input.shape == hyper_input.shape
|
||||
hyper_input_normed = normalized_input
|
||||
elif self.config.hc_per_branch_norm:
|
||||
hyper_input_normed = self.hc_norm(hyper_input)
|
||||
else:
|
||||
hyper_input_normed = self.hc_norm(
|
||||
@@ -272,10 +297,74 @@ class GatedResidual(HyperConnectionBase):
|
||||
self.hc_count,
|
||||
self.hidden_size,
|
||||
).to(self.params_dtype)
|
||||
return mixed_input, (hyper_input, hyper_input_normed)
|
||||
partials = None
|
||||
stream = self._gate_stream
|
||||
if (
|
||||
stream is not None
|
||||
and hyper_input.shape[0] == 1
|
||||
and hyper_input.dtype == torch.bfloat16
|
||||
and not torch.compiler.is_compiling()
|
||||
):
|
||||
from sglang.kernels.ops.elementwise.hc_combine_decode import hc_combine_gate
|
||||
|
||||
def combine(self, block_output: torch.Tensor, residuals) -> torch.Tensor:
|
||||
hyper_input, hyper_input_normed = residuals
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
partials = hc_combine_gate(
|
||||
hyper_input_normed, self.block_inject_weight.weight
|
||||
)
|
||||
hyper_input_normed.record_stream(stream)
|
||||
return mixed_input, (hyper_input, hyper_input_normed, partials)
|
||||
|
||||
def combine_and_normalize(
|
||||
self,
|
||||
block_output: torch.Tensor,
|
||||
residuals: tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]],
|
||||
next_norm: Optional[GroupedGemmaRMSNorm],
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Return the residual and its next HC normalization explicitly.
|
||||
|
||||
The BF16 combine result is retained before calculating the next norm.
|
||||
Callers must only pass the norm when no PLE or other update intervenes.
|
||||
"""
|
||||
hyper_input, _, partials = residuals
|
||||
if (
|
||||
partials is not None
|
||||
and next_norm is not None
|
||||
and next_norm.group_size == self.hidden_size
|
||||
and next_norm.weight.dtype == torch.bfloat16
|
||||
and block_output.dtype == torch.bfloat16
|
||||
):
|
||||
from sglang.kernels.ops.elementwise.hc_combine_decode import (
|
||||
hc_combine_apply_norm,
|
||||
)
|
||||
|
||||
current = torch.cuda.current_stream()
|
||||
current.wait_stream(self._gate_stream)
|
||||
partials.record_stream(current)
|
||||
return hc_combine_apply_norm(
|
||||
block_output,
|
||||
hyper_input,
|
||||
partials,
|
||||
next_norm.weight,
|
||||
next_norm.variance_epsilon,
|
||||
)
|
||||
return self.combine(block_output, residuals), None
|
||||
|
||||
def combine(
|
||||
self,
|
||||
block_output: torch.Tensor,
|
||||
residuals: tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]],
|
||||
) -> torch.Tensor:
|
||||
hyper_input, hyper_input_normed, partials = residuals
|
||||
if partials is not None:
|
||||
from sglang.kernels.ops.elementwise.hc_combine_decode import (
|
||||
hc_combine_apply,
|
||||
)
|
||||
|
||||
current = torch.cuda.current_stream()
|
||||
current.wait_stream(self._gate_stream)
|
||||
partials.record_stream(current)
|
||||
return hc_combine_apply(block_output, hyper_input, partials)
|
||||
assert hyper_input.shape[-1] == self.hc_count * self.hidden_size
|
||||
assert block_output.shape[-1] == self.hidden_size
|
||||
if block_output.shape[0] == 0:
|
||||
|
||||
@@ -100,10 +100,14 @@ class ExitPolicy:
|
||||
hidden_states: torch.Tensor,
|
||||
stream: ResidualStream,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
already_reduced: bool = False,
|
||||
):
|
||||
"""Complete the output of the operation-scheduled producer path, which
|
||||
never defers: the sum it owes, then its move and write-back."""
|
||||
steps = self.plan.path_for(forward_batch)
|
||||
if already_reduced and steps.output_move_completes_sum:
|
||||
raise ValueError("already-reduced output cannot use a reduce-scatter move")
|
||||
hidden_states, residual = self._complete_now(
|
||||
hidden_states,
|
||||
stream.residual,
|
||||
@@ -111,7 +115,8 @@ class ExitPolicy:
|
||||
dp_step=None,
|
||||
steps=steps,
|
||||
output_move=steps.output_move,
|
||||
owes_sum=steps.output.group is not None
|
||||
owes_sum=not already_reduced
|
||||
and steps.output.group is not None
|
||||
and not steps.output_move_completes_sum,
|
||||
)
|
||||
return self._record_output(
|
||||
|
||||
@@ -29,11 +29,11 @@ class GatedResidualState:
|
||||
per-stream gates, and writes its output back scaled by an injection
|
||||
coefficient computed from that same normalized residual.
|
||||
|
||||
Unlike the hyper-connection streams in mhc.py and ihc.py, whose read
|
||||
produces the coefficient its write-back consumes, the coefficient here is
|
||||
computed at write time. What has to survive from a stage's read to its
|
||||
write-back is therefore the normalized residual itself, which this state
|
||||
holds. Parameters belong to the owning layer's modules.
|
||||
The read carries its normalized residual to write-back. Decode may also
|
||||
prepare gate partials during the read; otherwise write-back computes the
|
||||
coefficient from that normalized residual. Both follow the residual rows
|
||||
through any slicing and are cleared after the FFN write-back.
|
||||
Parameters belong to the owning layer's modules.
|
||||
|
||||
Args:
|
||||
expand: Widens the layer stack's input into the stream representation,
|
||||
@@ -56,6 +56,7 @@ class GatedResidualState:
|
||||
attn_reads_every_row: bool = False
|
||||
# Produced by a stage's read and consumed by that same stage's write-back.
|
||||
normed: Optional[torch.Tensor] = None
|
||||
gate_partials: Optional[torch.Tensor] = None
|
||||
|
||||
def _read(self, mix, residual, out_norm):
|
||||
if out_norm is not None:
|
||||
@@ -63,7 +64,8 @@ class GatedResidualState:
|
||||
"a gated hyper-connection read with a separate norm; the read "
|
||||
"normalizes the streams itself"
|
||||
)
|
||||
hidden_states, (residual, self.normed) = mix(residual)
|
||||
hidden_states, residuals = mix(residual)
|
||||
residual, self.normed, self.gate_partials = residuals
|
||||
return hidden_states, residual
|
||||
|
||||
def read_attn_input(self, residual, out_norm=None):
|
||||
@@ -74,18 +76,25 @@ class GatedResidualState:
|
||||
return self._read(self.ffn_mix, residual, out_norm)
|
||||
|
||||
def apply_attn_combine(self, hidden_states, residual):
|
||||
return self.attn_combine(hidden_states, (residual, self.normed))
|
||||
return self.attn_combine(
|
||||
hidden_states, (residual, self.normed, self.gate_partials)
|
||||
)
|
||||
|
||||
def apply_ffn_combine(self, hidden_states, residual):
|
||||
return self.ffn_combine(hidden_states, (residual, self.normed))
|
||||
return self.ffn_combine(
|
||||
hidden_states, (residual, self.normed, self.gate_partials)
|
||||
)
|
||||
|
||||
def clear_coefficients(self):
|
||||
self.normed = None
|
||||
self.gate_partials = None
|
||||
|
||||
def slice_residual_attn_tp(self, residual):
|
||||
parallel = get_parallel()
|
||||
rank, size = parallel.attn_tp_rank, parallel.attn_tp_size
|
||||
self.normed = self.normed.tensor_split(size)[rank]
|
||||
if self.gate_partials is not None:
|
||||
self.gate_partials = self.gate_partials.tensor_split(size)[rank]
|
||||
return residual.tensor_split(size)[rank]
|
||||
|
||||
def gather_residual_attn_tp(self, residual):
|
||||
|
||||
@@ -262,12 +262,17 @@ class StageBoundary:
|
||||
hidden_states = stream.complete(hidden_states)
|
||||
return hidden_states, stream.snapshot(hidden_states)
|
||||
|
||||
def complete_now(self, hidden_states, forward_batch):
|
||||
"""Complete an FFN output outside exit() (the operation-scheduled TBO
|
||||
path, which never defers): the sum it owes, then the move onto the
|
||||
rows the layer hands on."""
|
||||
def complete_now(self, hidden_states, forward_batch, *, already_reduced=False):
|
||||
"""Complete an FFN output outside exit(), including its residual update.
|
||||
|
||||
already_reduced is for a producer kernel that fused the output sum.
|
||||
Row movement and residual write-back still belong to this boundary.
|
||||
"""
|
||||
return self.plan.output.complete_now(
|
||||
hidden_states, stream_of(forward_batch), forward_batch
|
||||
hidden_states,
|
||||
stream_of(forward_batch),
|
||||
forward_batch,
|
||||
already_reduced=already_reduced,
|
||||
)
|
||||
|
||||
def sum_part(self, hidden_states, forward_batch, group: SumGroup):
|
||||
|
||||
@@ -509,9 +509,16 @@ class FusedMoE(torch.nn.Module):
|
||||
and isinstance(self.quant_method, Fp8MoEMethod)
|
||||
and self.quant_method.block_quant
|
||||
)
|
||||
qwen4_bf16_deferred = (
|
||||
isinstance(self.quant_method, UnquantizedFusedMoEMethod)
|
||||
and params_dtype == torch.bfloat16
|
||||
and hidden_size == 2560
|
||||
and num_experts == 512
|
||||
and top_k == 10
|
||||
)
|
||||
self.supports_deferred_finalize = (
|
||||
get_moe_runner_backend().is_flashinfer_trtllm()
|
||||
and (nvfp4_deferred or qwen35_fp8_deferred)
|
||||
and (nvfp4_deferred or qwen35_fp8_deferred or qwen4_bf16_deferred)
|
||||
)
|
||||
global _deferred_finalize_info_logged
|
||||
if not _deferred_finalize_info_logged:
|
||||
|
||||
@@ -1640,6 +1640,9 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
|
||||
|
||||
hidden_states = dispatch_output.hidden_states
|
||||
topk_output = dispatch_output.topk_output
|
||||
defer_finalize = _deferred_finalize_enabled.get()
|
||||
if defer_finalize and use_routed_topk:
|
||||
raise RuntimeError("BF16 deferred finalize requires bypassed TopK")
|
||||
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group, disabled=not is_allocation_symmetric()
|
||||
@@ -1697,7 +1700,12 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
|
||||
routed_scaling_factor=runner_config.routed_scaling_factor,
|
||||
tune_max_num_tokens=next_power_of_2(hidden_states.shape[0]),
|
||||
activation_type=activation_type,
|
||||
do_finalize=not defer_finalize,
|
||||
)
|
||||
if defer_finalize:
|
||||
final_hidden_states = _make_deferred_finalize_output(
|
||||
final_hidden_states, top_k=topk_config.top_k
|
||||
)
|
||||
|
||||
return StandardCombineInput(hidden_states=final_hidden_states)
|
||||
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Single-token BF16 Qwen4 MoE finalize, shared gate and TP4 collective."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.communication.all_reduce_fusion import (
|
||||
moe_finalize_shared_gate_all_reduce,
|
||||
)
|
||||
from sglang.kernels.ops.communication.mp import register_comm_cleanup
|
||||
from sglang.kernels.ops.moe.shared_expert_gate import shared_expert_gate
|
||||
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
|
||||
CustomAllReduceV2,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_enable_moe_cp_allgather
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.runtime_context import get_exec, get_lora, get_parallel, get_spec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.models.qwen2_moe import Qwen2MoeSparseMoeBlock
|
||||
|
||||
|
||||
@lru_cache(None)
|
||||
def _get_decode_comm(tp_comm: CustomAllReduceV2) -> CustomAllReduceV2 | None:
|
||||
comm = CustomAllReduceV2(
|
||||
tp_comm.group,
|
||||
tp_comm.device,
|
||||
max_pull_size=0,
|
||||
max_pull_blocks=0,
|
||||
max_push_size=4 * 1024 * 1024,
|
||||
max_push_blocks=512,
|
||||
)
|
||||
if comm.disabled:
|
||||
return None
|
||||
register_comm_cleanup(comm)
|
||||
return comm
|
||||
|
||||
|
||||
def prepare_qwen4_decode_comm(
|
||||
mlp: Qwen2MoeSparseMoeBlock,
|
||||
) -> CustomAllReduceV2 | None:
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
parallel.tp_size != 4
|
||||
or parallel.attn_dp_size != 1
|
||||
or parallel.moe_ep_size != 1
|
||||
or get_lora().enable_lora
|
||||
or get_spec().speculative_algorithm is not None
|
||||
or get_exec().overlap.enable_two_batch_overlap
|
||||
or is_enable_moe_cp_allgather()
|
||||
or not mlp.supports_deferred_finalize
|
||||
or mlp.num_experts != 512
|
||||
or mlp.experts.w13_weight.dtype != torch.bfloat16
|
||||
or mlp.shared_expert_gate is None
|
||||
or mlp.enable_shared_expert_fusion
|
||||
or not torch.cuda.is_available()
|
||||
or torch.cuda.get_device_capability()[0] != 10
|
||||
):
|
||||
return None
|
||||
tp_comm = parallel.tp_group.ca_comm
|
||||
if not isinstance(tp_comm, CustomAllReduceV2) or tp_comm.disabled:
|
||||
return None
|
||||
return _get_decode_comm(tp_comm)
|
||||
|
||||
|
||||
def qwen4_decode_moe(
|
||||
hidden: torch.Tensor, mlp: Qwen2MoeSparseMoeBlock, comm: CustomAllReduceV2
|
||||
) -> torch.Tensor:
|
||||
if mlp.alt_stream is not None and get_is_capture_mode():
|
||||
current = torch.cuda.current_stream()
|
||||
mlp.alt_stream.wait_stream(current)
|
||||
# FC2 stays on the consuming stream for its PDL dependency. Both
|
||||
# branches read the input and write independently allocated outputs.
|
||||
with torch.cuda.stream(mlp.alt_stream):
|
||||
gate = shared_expert_gate(hidden, mlp.shared_expert_gate.weight)
|
||||
shared = mlp._forward_shared_experts(hidden, apply_gate=False)
|
||||
deferred = mlp._forward_router_experts(hidden, defer_finalize=True)
|
||||
current.wait_stream(mlp.alt_stream)
|
||||
gate.record_stream(current)
|
||||
shared.record_stream(current)
|
||||
else:
|
||||
gate = shared_expert_gate(hidden, mlp.shared_expert_gate.weight)
|
||||
shared = mlp._forward_shared_experts(hidden, apply_gate=False)
|
||||
deferred = mlp._forward_router_experts(hidden, defer_finalize=True)
|
||||
return moe_finalize_shared_gate_all_reduce(
|
||||
deferred.gemm2_out,
|
||||
deferred.expanded_idx_to_permuted_idx,
|
||||
deferred.expert_weights,
|
||||
shared,
|
||||
gate,
|
||||
comm.obj,
|
||||
)
|
||||
@@ -1,7 +1,8 @@
|
||||
"""Inference-only Qwen4-Exp (text + VL) on the Qwen3.5 backbone."""
|
||||
|
||||
import math
|
||||
from contextlib import nullcontext
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable, Optional, Set, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
@@ -34,6 +35,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
)
|
||||
from sglang.srt.layers.hyperconnection import (
|
||||
GatedResidual,
|
||||
GroupedGemmaRMSNorm,
|
||||
HyperConnectionConfig,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
@@ -47,6 +49,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.modelopt_quant import (
|
||||
ModelOptMixedPrecisionConfig,
|
||||
@@ -1422,11 +1425,24 @@ class _MixStreams:
|
||||
"""The layer stack's terminal read: the gated mix down to one stream. The
|
||||
stack ends on the mix, with no final norm after it."""
|
||||
|
||||
def __init__(self, hyper_connection_mixer):
|
||||
def __init__(self, hyper_connection_mixer, normalized_input=None):
|
||||
self.hyper_connection_mixer = hyper_connection_mixer
|
||||
self.normalized_input = normalized_input
|
||||
|
||||
def __call__(self, hidden_states):
|
||||
return self.hyper_connection_mixer.mix(hidden_states)[0]
|
||||
return self.hyper_connection_mixer.mix(
|
||||
hidden_states, normalized_input=self.normalized_input
|
||||
)[0]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _HCNorms:
|
||||
"""Prepared norms for one layer invocation; never carried across PLE or PP."""
|
||||
|
||||
attn_input: Optional[torch.Tensor]
|
||||
next_norm: Optional[GroupedGemmaRMSNorm]
|
||||
ffn_input: Optional[torch.Tensor] = None
|
||||
output: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
def _has_ple(layer_id, config) -> bool:
|
||||
@@ -1470,6 +1486,7 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
self.ple = None
|
||||
self._ple_forward_batch = None
|
||||
self._ple_batch = None
|
||||
self._hc_norms = None
|
||||
|
||||
# The gated reads normalize the streams themselves, so these layers
|
||||
# have neither of the norms their base class builds.
|
||||
@@ -1515,15 +1532,41 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
GatedResidualState(
|
||||
expand=self._widen_streams,
|
||||
attn_mix=self._attn_mix,
|
||||
ffn_mix=self.mlp_hyper_connection.mix,
|
||||
attn_combine=self.attn_hyper_connection.combine,
|
||||
ffn_combine=self.mlp_hyper_connection.combine,
|
||||
ffn_mix=self._ffn_mix,
|
||||
attn_combine=self._attn_combine,
|
||||
ffn_combine=self._ffn_combine,
|
||||
# The PLE embedding is computed for every row.
|
||||
attn_reads_every_row=self.ple is not None,
|
||||
).residual_ops(),
|
||||
sparse=isinstance(self.mlp, Qwen2MoeSparseMoeBlock),
|
||||
)
|
||||
|
||||
from sglang.srt.layers.moe.qwen4_decode import prepare_qwen4_decode_comm
|
||||
|
||||
self._decode_moe_comm = (
|
||||
prepare_qwen4_decode_comm(self.mlp)
|
||||
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock)
|
||||
else None
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def hc_norms(self, normalized_input, next_norm):
|
||||
norms = _HCNorms(normalized_input, next_norm)
|
||||
# A prepared norm can only follow its residual without row movement.
|
||||
self._hc_norms = (
|
||||
norms
|
||||
if get_parallel().attn_dp_size == 1
|
||||
and get_parallel().attn_cp_size == 1
|
||||
and get_parallel().moe_ep_size == 1
|
||||
and not get_attn_tp_context().input_scattered
|
||||
and get_moe_a2a_backend().is_none()
|
||||
else None
|
||||
)
|
||||
try:
|
||||
yield norms
|
||||
finally:
|
||||
self._hc_norms = None
|
||||
|
||||
def _widen_streams(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Open the layer stack's input into one stream per hyper connection.
|
||||
An MTP draft is handed streams that are already widened."""
|
||||
@@ -1538,7 +1581,42 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
own contribution to the streams and joins them before the read."""
|
||||
if self.ple is not None:
|
||||
residual = self._add_ple_embedding(residual)
|
||||
return self.attn_hyper_connection.mix(residual)
|
||||
normalized_input = (
|
||||
self._hc_norms.attn_input
|
||||
if self._hc_norms is not None and self.ple is None
|
||||
else None
|
||||
)
|
||||
return self.attn_hyper_connection.mix(
|
||||
residual, normalized_input=normalized_input
|
||||
)
|
||||
|
||||
def _attn_combine(self, hidden_states, residuals):
|
||||
if self._hc_norms is None:
|
||||
return self.attn_hyper_connection.combine(hidden_states, residuals)
|
||||
hidden_states, self._hc_norms.ffn_input = (
|
||||
self.attn_hyper_connection.combine_and_normalize(
|
||||
hidden_states, residuals, self.mlp_hyper_connection.hc_norm
|
||||
)
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def _ffn_mix(self, residual):
|
||||
normalized_input = (
|
||||
self._hc_norms.ffn_input if self._hc_norms is not None else None
|
||||
)
|
||||
return self.mlp_hyper_connection.mix(
|
||||
residual, normalized_input=normalized_input
|
||||
)
|
||||
|
||||
def _ffn_combine(self, hidden_states, residuals):
|
||||
if self._hc_norms is None:
|
||||
return self.mlp_hyper_connection.combine(hidden_states, residuals)
|
||||
hidden_states, self._hc_norms.output = (
|
||||
self.mlp_hyper_connection.combine_and_normalize(
|
||||
hidden_states, residuals, self._hc_norms.next_norm
|
||||
)
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def _add_ple_embedding(self, residual: torch.Tensor) -> torch.Tensor:
|
||||
forward_batch, ple_batch = self._ple_forward_batch, self._ple_batch
|
||||
@@ -1573,6 +1651,21 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
if (
|
||||
self._decode_moe_comm is not None
|
||||
and hidden_states.shape[0] == 1
|
||||
and hidden_states.dtype == torch.bfloat16
|
||||
and not get_attn_tp_context().input_scattered
|
||||
and not torch.compiler.is_compiling()
|
||||
):
|
||||
from sglang.srt.layers.moe.qwen4_decode import qwen4_decode_moe
|
||||
|
||||
hidden_states = qwen4_decode_moe(
|
||||
hidden_states, self.mlp, self._decode_moe_comm
|
||||
)
|
||||
return self.ffn_boundary.complete_now(
|
||||
hidden_states, forward_batch, already_reduced=True
|
||||
)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
if isinstance(self.mlp, Qwen2MoeSparseMoeBlock):
|
||||
hidden_states = self.mlp(
|
||||
@@ -1855,8 +1948,6 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
if self.pp_group.is_last_rank
|
||||
else PPMissingLayer()
|
||||
)
|
||||
# The stack's terminal read mixes the streams down; there is no final norm.
|
||||
self._mix_streams = _MixStreams(self.hyper_connection_mixer)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -1903,6 +1994,7 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
ngram_eos_token_id=self.ple_ngram_eos_token_id,
|
||||
)
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
normalized_input = None
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
if i + 1 < self.end_layer:
|
||||
@@ -1914,7 +2006,17 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
next_ple.start_prefetch(
|
||||
batch=ple_batch, forward_batch=forward_batch
|
||||
)
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
next_norm = None
|
||||
if i + 1 < self.end_layer:
|
||||
next_layer = self.layers[i + 1]
|
||||
if next_layer.ple is None:
|
||||
next_norm = next_layer.attn_hyper_connection.hc_norm
|
||||
elif self.pp_group.is_last_rank:
|
||||
next_norm = self.hyper_connection_mixer.hc_norm
|
||||
with (
|
||||
get_global_expert_distribution_recorder().with_current_layer(i),
|
||||
layer.hc_norms(normalized_input, next_norm) as norms,
|
||||
):
|
||||
hidden_states = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
@@ -1927,6 +2029,8 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
),
|
||||
)
|
||||
|
||||
normalized_input = norms.output
|
||||
|
||||
if breakable_ple:
|
||||
_breakable_commit_ple_batch(batch=ple_batch)
|
||||
else:
|
||||
@@ -1942,7 +2046,9 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
|
||||
hc_hidden_states = hidden_states
|
||||
hidden_states = residual_batch.final_norm(
|
||||
hidden_states, forward_batch, self._mix_streams
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
_MixStreams(self.hyper_connection_mixer, normalized_input),
|
||||
)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
return hidden_states, hc_hidden_states
|
||||
|
||||
Reference in New Issue
Block a user