From 13a6dee99b10305f7acbd1f9094acd2df4ea26ef Mon Sep 17 00:00:00 2001 From: Monishver <79901686+Monishver11@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:32:50 -0400 Subject: [PATCH] [Fix] moe_sum_reduce: gate the bf16 vectorized path on hidden % 16 (#42837) Signed-off-by: Monishver Chandrasekaran Co-authored-by: Claude Fable 5.1 Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- .../sglang/kernels/aot/csrc/moe/moe_sum_reduce.cu | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python/sglang/kernels/aot/csrc/moe/moe_sum_reduce.cu b/python/sglang/kernels/aot/csrc/moe/moe_sum_reduce.cu index 791ce620b2..6748ea7611 100644 --- a/python/sglang/kernels/aot/csrc/moe/moe_sum_reduce.cu +++ b/python/sglang/kernels/aot/csrc/moe/moe_sum_reduce.cu @@ -53,6 +53,10 @@ union Pack16B { __nv_bfloat16 u16[8]; }; +// Elements per lane per iteration of the bf16 vectorized kernel; the launcher +// gates on this so a row the kernel cannot cover never reaches it. +constexpr int BF16_VEC = 16; + template __global__ void moe_sum_reduce_warp_per_token_vec_kernel( const at::BFloat16* __restrict__ x, @@ -64,7 +68,7 @@ __global__ void moe_sum_reduce_warp_per_token_vec_kernel( const int64_t stride_topk, // in elements const int64_t out_stride_token, // in elements const float scale) { - constexpr int VEC = 16; + constexpr int VEC = BF16_VEC; constexpr int PACKS = VEC / 8; const int warp_id = threadIdx.x / 32; @@ -245,14 +249,15 @@ void moe_sum_reduce(at::Tensor& input, at::Tensor& output, double routed_scaling auto stream = at::cuda::getCurrentCUDAStream(); - const bool fast_bf16_vec_ok = (input.scalar_type() == at::kBFloat16) && (token_num > 256) && (hidden_dim % 8 == 0); + const bool fast_bf16_vec_ok = + (input.scalar_type() == at::kBFloat16) && (token_num > 256) && (hidden_dim % BF16_VEC == 0); // Fast path for bf16 vectorize if (fast_bf16_vec_ok) { constexpr int WARPS_PER_BLOCK = 8; constexpr int THREADS = WARPS_PER_BLOCK * 32; - const int64_t n_chunks = hidden_dim / 8; + const int64_t n_chunks = hidden_dim / BF16_VEC; int64_t grid_x = (n_chunks + 32 - 1) / 32; if (grid_x > 65535) grid_x = 65535; @@ -262,8 +267,6 @@ void moe_sum_reduce(at::Tensor& input, at::Tensor& output, double routed_scaling dim3 block(THREADS); dim3 grid(static_cast(grid_x), static_cast(grid_y)); - auto stream = at::cuda::getCurrentCUDAStream(); - const float scale = static_cast(routed_scaling_factor); moe_sum_reduce_warp_per_token_vec_kernel<<>>( reinterpret_cast(input.data_ptr()),