From 8b9b992e417a950ad57883ee6903ef04d6b3fa5c Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:31:17 -0700 Subject: [PATCH] [Perf] moe: Preserve Marlin workspace and route-count budgets (#45462) --- .../kernels/jit/csrc/gemm/marlin_moe/kernel.h | 5 +- .../csrc/gemm/marlin_moe/marlin_template.h | 138 ++++++++-- .../csrc/gemm/marlin_moe/moe_wna16_marlin.cuh | 175 +++++++----- .../jit/csrc/moe/align_single_token.cuh | 18 +- .../kernels/jit/csrc/moe/moe_align_kernel.cu | 167 ++++++++++-- .../moe/trtllm_lora_temp/topk_softmax_pack.py | 5 + .../kernels/ops/lora/moe/virtual_experts.py | 23 +- python/sglang/kernels/ops/moe/moe_align.py | 33 ++- .../kernels/ops/moe/moe_align_single_token.py | 18 +- .../kernels/ops/moe/moe_align_small_numel.py | 17 +- .../kernels/ops/moe/moe_wna16_marlin.py | 39 ++- .../muse_spark_post_expert_finalize_triton.py | 25 +- .../moe/fused_moe_triton/fused_marlin_moe.py | 16 +- .../triton_utils/moe_align_block_size.py | 39 ++- .../layers/muse_spark/muse_spark_fp4_moe.py | 36 ++- .../ops/moe/test_marlin_packed_topk_unpack.py | 81 ++++++ .../ops/moe/test_moe_align_block_size.py | 224 ++++++++++++++- .../kernels/ops/moe/test_moe_wna16_marlin.py | 258 ++++++++++++++---- .../ops/moe/test_muse_spark_routing.py | 82 +++++- 19 files changed, 1173 insertions(+), 226 deletions(-) diff --git a/python/sglang/kernels/jit/csrc/gemm/marlin_moe/kernel.h b/python/sglang/kernels/jit/csrc/gemm/marlin_moe/kernel.h index ccc47e7392..2791daaf77 100644 --- a/python/sglang/kernels/jit/csrc/gemm/marlin_moe/kernel.h +++ b/python/sglang/kernels/jit/csrc/gemm/marlin_moe/kernel.h @@ -34,7 +34,10 @@ template < // with a separate quantization scale const bool is_zp_float, // is zero point of float16 type? const bool kIsEP, // expert parallelism - const bool kHasBias // has per-expert bias + const bool kHasBias, // has per-expert bias + const bool kExpertIdsAreSorted, // sorted block expert IDs + const bool kZeroInactiveOutput, // zero rows in inactive routing blocks + const bool kHasInactivePrefixCount // producer-published padded-token/prefix pair > __global__ void Marlin(MARLIN_KERNEL_PARAMS); diff --git a/python/sglang/kernels/jit/csrc/gemm/marlin_moe/marlin_template.h b/python/sglang/kernels/jit/csrc/gemm/marlin_moe/marlin_template.h index 010ff58ec3..720d28e63e 100644 --- a/python/sglang/kernels/jit/csrc/gemm/marlin_moe/marlin_template.h +++ b/python/sglang/kernels/jit/csrc/gemm/marlin_moe/marlin_template.h @@ -55,7 +55,10 @@ template < // with a separate quantization scale const bool is_zp_float, // is zero point of float16 type? const bool kIsEP, // expert parallelism - const bool kHasBias // has per-expert bias + const bool kHasBias, // has per-expert bias + const bool kExpertIdsAreSorted, // sorted block expert IDs + const bool kZeroInactiveOutput, // zero rows in inactive routing blocks + const bool kHasInactivePrefixCount // producer-published padded-token/prefix pair > __global__ void Marlin( const int4* __restrict__ A, // fp16 input matrix of shape mxk @@ -87,6 +90,27 @@ __global__ void Marlin( #else +template +__device__ __forceinline__ void zero_inactive_output_rows( + int4* __restrict__ output, + const int32_t* __restrict__ sorted_token_ids, + const int32_t* __restrict__ expert_ids, + int num_blocks, + int num_routes, + int row_vectors) { + for (int block = blockIdx.x; block < num_blocks; block += gridDim.x) { + if constexpr (!kExpertIdsAreSorted) { + if (expert_ids[block] != -1) continue; + } + for (int offset = threadIdx.x; offset < kMoeBlockSize * row_vectors; offset += blockDim.x) { + int route = sorted_token_ids[block * kMoeBlockSize + offset / row_vectors]; + if (route < num_routes) { + output[static_cast(route) * row_vectors + offset % row_vectors] = {0, 0, 0, 0}; + } + } + } +} + // m16n8k16 tensor core mma instruction with fp16 inputs and fp32 // output/accumulation. template @@ -298,7 +322,10 @@ template < // with a separate quantization scale const bool is_zp_float, // is zero point of float16 type? const bool kIsEP, // expert parallelism - const bool kHasBias // has per-expert bias + const bool kHasBias, // has per-expert bias + const bool kExpertIdsAreSorted, // sorted block expert IDs + const bool kZeroInactiveOutput, // zero rows in inactive routing blocks + const bool kHasInactivePrefixCount // producer-published padded-token/prefix pair > __global__ void Marlin( const int4* __restrict__ A, // fp16 input matrix of shape mxk @@ -381,19 +408,71 @@ __global__ void Marlin( is_zp_float ? prob_n * prob_k / group_size / 8 : prob_n * prob_k / group_size / (pack_factor * 4); const int b_bias_expert_stride = prob_n / 8; - int num_tokens_past_padded = num_tokens_past_padded_ptr[0]; + static_assert(!kHasInactivePrefixCount || (kExpertIdsAreSorted && !kIsEP)); + int num_tokens_past_padded; + int inactive_prefix_blocks = 0; + if constexpr (kHasInactivePrefixCount) { + // One aligned transaction publishes both quantities without a dependent head-ID load. + const int2 counts = __ldg(reinterpret_cast(num_tokens_past_padded_ptr)); + num_tokens_past_padded = counts.x; + inactive_prefix_blocks = counts.y; + } else { + num_tokens_past_padded = num_tokens_past_padded_ptr[0]; + } int parallel = num_tokens_past_padded / moe_block_size; - // Every warp counts the same inactive blocks without a serial per-thread scan. - int num_invalid_blocks = 0; - for (int i = threadIdx.x % 32; i < parallel; i += 32) { - num_invalid_blocks += expert_ids_ptr[i] == -1; - } -#pragma unroll - for (int offset = 16; offset > 0; offset /= 2) { - num_invalid_blocks += __shfl_xor_sync(0xffffffff, num_invalid_blocks, offset); - } - parallel -= num_invalid_blocks; if (parallel == 0) return; + int num_invalid_blocks = 0; + if constexpr (kHasInactivePrefixCount) { + if (inactive_prefix_blocks > 0) { + if constexpr (kZeroInactiveOutput) { + zero_inactive_output_rows( + C, sorted_token_ids_ptr, expert_ids_ptr, inactive_prefix_blocks, prob_m * top_k, prob_n / 8); + } + parallel -= inactive_prefix_blocks; + if (parallel == 0) return; + expert_ids_ptr += inactive_prefix_blocks; + sorted_token_ids_ptr += inactive_prefix_blocks * moe_block_size; + } + } else if constexpr (kExpertIdsAreSorted) { + if (expert_ids_ptr[0] < 0) { + // Only the used prefix is sorted; allocated tail entries are undefined. + int first = 0; + int last = parallel; + while (first < last) { + int middle = first + (last - first) / 2; + if (expert_ids_ptr[middle] < 0) + first = middle + 1; + else + last = middle; + } + if constexpr (kZeroInactiveOutput) { + // Preserve the original route table until its inactive destinations are written. + zero_inactive_output_rows( + C, sorted_token_ids_ptr, expert_ids_ptr, first, prob_m * top_k, prob_n / 8); + } + parallel -= first; + if (parallel == 0) return; + expert_ids_ptr += first; + sorted_token_ids_ptr += first * moe_block_size; + } + } else { + // Generic and EP callers may have scattered inactive blocks. + for (int i = threadIdx.x % 32; i < parallel; i += 32) { + num_invalid_blocks += expert_ids_ptr[i] == -1; + } +#pragma unroll + for (int offset = 16; offset > 0; offset /= 2) { + num_invalid_blocks += __shfl_xor_sync(0xffffffff, num_invalid_blocks, offset); + } + if constexpr (kZeroInactiveOutput) { + if (num_invalid_blocks > 0) { + zero_inactive_output_rows( + C, sorted_token_ids_ptr, expert_ids_ptr, parallel, prob_m * top_k, prob_n / 8); + } + } + parallel -= num_invalid_blocks; + if (parallel == 0) return; + } int k_tiles = prob_k / 16 / thread_k_blocks; int n_tiles = prob_n / 16 / thread_n_blocks; @@ -521,22 +600,27 @@ __global__ void Marlin( if (par_id >= parallel) return; old_expert_id = expert_id; - if (num_invalid_blocks > 0) { - int skip_count = block_id == -1 ? par_id : 0; - block_id++; - for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) { - expert_id = expert_ids_ptr[i]; - if (expert_id != -1) { - if (skip_count == 0) { - block_id = i; - break; - }; - skip_count--; - }; - } - } else { + if constexpr (kExpertIdsAreSorted) { block_id = par_id; expert_id = expert_ids_ptr[block_id]; + } else { + if (num_invalid_blocks > 0) { + int skip_count = block_id == -1 ? par_id : 0; + block_id++; + for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) { + expert_id = expert_ids_ptr[i]; + if (expert_id != -1) { + if (skip_count == 0) { + block_id = i; + break; + }; + skip_count--; + }; + } + } else { + block_id = par_id; + expert_id = expert_ids_ptr[block_id]; + } } if constexpr (w_type == host::kFE2M1f && s_type == host::kFE4M3fn) { diff --git a/python/sglang/kernels/jit/csrc/gemm/marlin_moe/moe_wna16_marlin.cuh b/python/sglang/kernels/jit/csrc/gemm/marlin_moe/moe_wna16_marlin.cuh index 87647915df..2e0ef2bd0b 100644 --- a/python/sglang/kernels/jit/csrc/gemm/marlin_moe/moe_wna16_marlin.cuh +++ b/python/sglang/kernels/jit/csrc/gemm/marlin_moe/moe_wna16_marlin.cuh @@ -315,7 +315,10 @@ bool is_valid_config( GROUP_BLOCKS, \ IS_ZP_FLOAT, \ kIsEP, \ - kHasBias>; \ + kHasBias, \ + kExpertIdsAreSorted, \ + kZeroInactiveOutput, \ + kHasInactivePrefixCount>; \ } // COMMON: cases for (group_blocks in [-1, 2, 4, 8] and is_zp_float == false) @@ -440,7 +443,13 @@ bool is_valid_config( ACT_GET_IF_M234(W_TYPE, 16, 4, 256) \ ACT_GET_IF_M234(W_TYPE, 8, 4, 128) -template +template < + typename scalar_t, + bool kIsEP, + bool kHasBias, + bool kExpertIdsAreSorted, + bool kZeroInactiveOutput, + bool kHasInactivePrefixCount> MarlinFuncPtr get_marlin_kernel( const host::ScalarType q_type, int thread_m_blocks, @@ -476,7 +485,13 @@ MarlinFuncPtr get_marlin_kernel( return kernel; } -template +template < + typename scalar_t, + bool kIsEP, + bool kHasBias, + bool kExpertIdsAreSorted, + bool kZeroInactiveOutput, + bool kHasInactivePrefixCount> exec_config_t determine_exec_config( const host::ScalarType& q_type, int prob_m, @@ -549,17 +564,18 @@ exec_config_t determine_exec_config( group_blocks = group_size == -1 ? -1 : (group_size / 16); } - auto kernel = get_marlin_kernel( - q_type, - thread_m_blocks, - th_config.thread_n / 16, - th_config.thread_k / 16, - m_block_size_8, - has_act_order, - has_zp, - group_blocks, - th_config.num_threads, - is_zp_float); + auto kernel = + get_marlin_kernel( + q_type, + thread_m_blocks, + th_config.thread_n / 16, + th_config.thread_k / 16, + m_block_size_8, + has_act_order, + has_zp, + group_blocks, + th_config.num_threads, + is_zp_float); if (kernel == MarlinDefault) continue; @@ -587,7 +603,13 @@ exec_config_t determine_exec_config( return exec_cfg; } -template +template < + typename scalar_t, + bool kIsEP, + bool kHasBias, + bool kExpertIdsAreSorted, + bool kZeroInactiveOutput, + bool kHasInactivePrefixCount> void marlin_mm( const void* A, const void* B, @@ -727,7 +749,13 @@ void marlin_mm( host::RuntimeCheck(prob_k % thread_k == 0, "prob_k = ", prob_k, " is not divisible by thread_k = ", thread_k); } else { // Auto config - exec_cfg = determine_exec_config( + exec_cfg = determine_exec_config< + scalar_t, + kIsEP, + kHasBias, + kExpertIdsAreSorted, + kZeroInactiveOutput, + kHasInactivePrefixCount>( q_type, prob_m, prob_n, @@ -799,17 +827,18 @@ void marlin_mm( ", max_shared_mem = ", max_shared_mem); - auto kernel = get_marlin_kernel( - q_type, - thread_m_blocks, - thread_n_blocks, - thread_k_blocks, - m_block_size_8, - has_act_order, - has_zp, - group_blocks, - num_threads, - is_zp_float); + auto kernel = + get_marlin_kernel( + q_type, + thread_m_blocks, + thread_n_blocks, + thread_k_blocks, + m_block_size_8, + has_act_order, + has_zp, + group_blocks, + num_threads, + is_zp_float); if (kernel == MarlinDefault) { host::Panic( @@ -850,7 +879,13 @@ void marlin_mm( } // namespace device::marlin_moe -template +template < + typename scalar_t, + bool kIsEP, + bool kHasBias, + bool kExpertIdsAreSorted, + bool kZeroInactiveOutput, + bool kHasInactivePrefixCount> void moe_wna16_marlin_gemm( tvm::ffi::TensorView a, tvm::ffi::TensorView c, @@ -887,6 +922,8 @@ void moe_wna16_marlin_gemm( bool is_zp_float) { using namespace host; + static_assert(!kIsEP || !kExpertIdsAreSorted, "Sorted block IDs are specialized only without EP"); + static_assert(!kHasInactivePrefixCount || (kExpertIdsAreSorted && !kIsEP)); RuntimeCheck(is_ep == kIsEP, "is_ep does not match the compiled Marlin MoE specialization"); RuntimeCheck(has_bias == kHasBias, "has_bias does not match the compiled Marlin MoE specialization"); @@ -930,6 +967,9 @@ void moe_wna16_marlin_gemm( auto device = SymbolicDevice{}; device.set_options(); TensorMatcher({-1, -1}).with_dtype().with_device(device).verify(a); + if constexpr (kHasInactivePrefixCount) { + TensorMatcher({2}).with_dtype().with_device(device).ensure_alignment(8).verify(num_tokens_post_padded); + } device.verify(b_q_weight.device()); RuntimeCheck(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); @@ -1087,45 +1127,46 @@ void moe_wna16_marlin_gemm( // Early return for zero-size M (moved after all validation) if (size_m == 0) return; - device::marlin_moe::marlin_mm( - a.data_ptr(), - b_q_weight.data_ptr(), - c.data_ptr(), - c_tmp.data_ptr(), - b_bias.data_ptr(), - b_scales.data_ptr(), - global_scale.data_ptr(), - b_zeros.data_ptr(), - g_idx.data_ptr(), - perm.data_ptr(), - a_tmp.data_ptr(), - sorted_token_ids.data_ptr(), - expert_ids.data_ptr(), - num_tokens_post_padded.data_ptr(), - topk_weights.data_ptr(), - static_cast(moe_block_size), - static_cast(top_k), - mul_topk_weights, - is_ep, - static_cast(size_m), - static_cast(size_n), - static_cast(size_k), - workspace.data_ptr(), - b_q_type, - has_bias, - has_act_order, - is_k_full, - has_zp, - static_cast(num_groups), - static_cast(group_size), - dev, - stream, - thread_k, - thread_n, - sms, - use_atomic_add, - use_fp32_reduce, - is_zp_float); + device::marlin_moe:: + marlin_mm( + a.data_ptr(), + b_q_weight.data_ptr(), + c.data_ptr(), + c_tmp.data_ptr(), + b_bias.data_ptr(), + b_scales.data_ptr(), + global_scale.data_ptr(), + b_zeros.data_ptr(), + g_idx.data_ptr(), + perm.data_ptr(), + a_tmp.data_ptr(), + sorted_token_ids.data_ptr(), + expert_ids.data_ptr(), + num_tokens_post_padded.data_ptr(), + topk_weights.data_ptr(), + static_cast(moe_block_size), + static_cast(top_k), + mul_topk_weights, + is_ep, + static_cast(size_m), + static_cast(size_n), + static_cast(size_k), + workspace.data_ptr(), + b_q_type, + has_bias, + has_act_order, + is_k_full, + has_zp, + static_cast(num_groups), + static_cast(group_size), + dev, + stream, + thread_k, + thread_n, + sms, + use_atomic_add, + use_fp32_reduce, + is_zp_float); } } // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/moe/align_single_token.cuh b/python/sglang/kernels/jit/csrc/moe/align_single_token.cuh index 6ab4709d42..4bc3d009b7 100644 --- a/python/sglang/kernels/jit/csrc/moe/align_single_token.cuh +++ b/python/sglang/kernels/jit/csrc/moe/align_single_token.cuh @@ -23,12 +23,12 @@ struct AlignSingleTokenParams { const int32_t* __restrict__ topk_ids; // [1, topk] int32_t* __restrict__ sorted_ids; // [topk * block_size] int32_t* __restrict__ expert_ids; // [topk] - int32_t* __restrict__ num_post; // [1] + int32_t* __restrict__ num_post; // [1] or [2] uint32_t topk; uint32_t block_size; }; -template +template __global__ void align_single_token_kernel(const AlignSingleTokenParams __grid_constant__ params) { using namespace device; const uint32_t lane = threadIdx.x; // one warp @@ -61,16 +61,21 @@ __global__ void align_single_token_kernel(const AlignSingleTokenParams __grid_co params.sorted_ids[p] = static_cast(topk); } } - if (lane == 0) { + if constexpr (kWriteInactivePrefixCount) { + const uint32_t inactive = __ballot_sync(0xffffffff, lane < topk && my_id < 0); + if (lane == 0) { + *reinterpret_cast(params.num_post) = make_int2(topk * bs, __popc(inactive)); + } + } else if (lane == 0) { params.num_post[0] = static_cast(topk * bs); } PDLTriggerSecondary(); } -template +template struct AlignSingleTokenKernel { - static constexpr auto kernel = align_single_token_kernel; + static constexpr auto kernel = align_single_token_kernel; static void run(const tvm::ffi::TensorView topk_ids, @@ -86,6 +91,9 @@ struct AlignSingleTokenKernel { device.set_options(); TensorMatcher({One_, K_}).with_dtype().with_device(device).verify(topk_ids); + if constexpr (kWriteInactivePrefixCount) { + TensorMatcher({2}).with_dtype().with_device(device).ensure_alignment(8).verify(num_post); + } const auto topk = static_cast(K_.unwrap()); RuntimeCheck(One_.unwrap() == 1, "moe_align_single_token requires M == 1"); diff --git a/python/sglang/kernels/jit/csrc/moe/moe_align_kernel.cu b/python/sglang/kernels/jit/csrc/moe/moe_align_kernel.cu index ec72b4a4fe..79b393abb8 100644 --- a/python/sglang/kernels/jit/csrc/moe/moe_align_kernel.cu +++ b/python/sglang/kernels/jit/csrc/moe/moe_align_kernel.cu @@ -23,6 +23,7 @@ limitations under the License. #include #include +#include #ifndef WARP_SIZE #define WARP_SIZE 32 @@ -37,6 +38,14 @@ using Vec = int4; namespace moe { +template +__device__ __forceinline__ int32_t sentinel_expert_bucket(scalar_t expert_id, int32_t num_buckets) { + static_assert(std::is_same_v || std::is_same_v); + using unsigned_t = std::make_unsigned_t; + const unsigned_t shifted = static_cast(expert_id) + unsigned_t{1}; + return shifted < static_cast(num_buckets) ? static_cast(shifted) : 0; +} + template __global__ void count_and_sort_expert_tokens_kernel( const scalar_t* __restrict__ topk_ids, @@ -53,7 +62,24 @@ __global__ void count_and_sort_expert_tokens_kernel( } } -template +template +__global__ void count_and_sort_checked_expert_tokens_kernel( + const scalar_t* __restrict__ topk_ids, + int32_t* __restrict__ sorted_token_ids, + int32_t* __restrict__ cumsum_buffer, + size_t numel, + int32_t num_buckets) { + const size_t tid = blockIdx.x * blockDim.x + threadIdx.x; + const size_t stride = blockDim.x * gridDim.x; + + for (size_t i = tid; i < numel; i += stride) { + const int32_t expert_id = sentinel_expert_bucket(topk_ids[i], num_buckets); + const int32_t rank_post_pad = atomicAdd(&cumsum_buffer[expert_id], 1); + sorted_token_ids[rank_post_pad] = i; + } +} + +template __global__ void moe_align_block_size_kernel( const scalar_t* __restrict__ topk_ids, int32_t* __restrict__ sorted_token_ids, @@ -101,7 +127,12 @@ __global__ void moe_align_block_size_kernel( __syncthreads(); for (size_t i = tid; i < numel; i += stride) { - int expert_id = topk_ids[i] + 1; + int expert_id; + if constexpr (kMapInvalidExpertsToSentinel) { + expert_id = sentinel_expert_bucket(topk_ids[i], num_experts); + } else { + expert_id = topk_ids[i] + 1; + } atomicAdd(&shared_counts[expert_id], 1); } @@ -163,7 +194,12 @@ __global__ void moe_align_block_size_kernel( if (tid == 0) { s_total_tokens_post_pad = prefix[num_experts]; - *total_tokens_post_pad = s_total_tokens_post_pad; + if constexpr (kWriteInactivePrefixCount) { + *reinterpret_cast(total_tokens_post_pad) = + make_int2(s_total_tokens_post_pad, CEILDIV(shared_counts[0], block_size)); + } else { + *total_tokens_post_pad = s_total_tokens_post_pad; + } } __syncthreads(); @@ -190,7 +226,12 @@ __global__ void moe_align_block_size_kernel( if (tid == 0) { prefix[num_experts] = warp_sums[num_warps_for_scan - 1]; s_total_tokens_post_pad = prefix[num_experts]; - *total_tokens_post_pad = s_total_tokens_post_pad; + if constexpr (kWriteInactivePrefixCount) { + *reinterpret_cast(total_tokens_post_pad) = + make_int2(s_total_tokens_post_pad, CEILDIV(shared_counts[0], block_size)); + } else { + *total_tokens_post_pad = s_total_tokens_post_pad; + } } __syncthreads(); @@ -241,7 +282,12 @@ __global__ void moe_align_block_size_kernel( } } -template +template < + typename scalar_t, + int32_t fill_threads, + bool kWriteInactivePrefixCount, + bool kMapInvalidExpertsToSentinel, + bool kPadExpertIds> __global__ void moe_align_block_size_small_batch_expert_kernel( const scalar_t* __restrict__ topk_ids, int32_t* __restrict__ sorted_token_ids, @@ -285,7 +331,12 @@ __global__ void moe_align_block_size_small_batch_expert_kernel( } for (size_t i = tid; i < numel; i += stride) { - int32_t expert_id = topk_ids[i] + 1; + int32_t expert_id; + if constexpr (kMapInvalidExpertsToSentinel) { + expert_id = sentinel_expert_bucket(topk_ids[i], num_experts); + } else { + expert_id = topk_ids[i] + 1; + } ++tokens_cnts[(tid + 1) * num_experts + expert_id]; } @@ -305,7 +356,11 @@ __global__ void moe_align_block_size_small_batch_expert_kernel( for (int i = 1; i <= num_experts; ++i) { cumsum[i] = cumsum[i - 1] + CEILDIV(tokens_cnts[stride * num_experts + i - 1], block_size) * block_size; } - *total_tokens_post_pad = static_cast(cumsum[num_experts]); + if constexpr (kWriteInactivePrefixCount) { + *reinterpret_cast(total_tokens_post_pad) = make_int2(cumsum[num_experts], cumsum[1] / block_size); + } else { + *total_tokens_post_pad = static_cast(cumsum[num_experts]); + } } __syncthreads(); @@ -322,7 +377,12 @@ __global__ void moe_align_block_size_small_batch_expert_kernel( } for (size_t i = tid; i < numel; i += stride) { - int32_t expert_id = topk_ids[i] + 1; + int32_t expert_id; + if constexpr (kMapInvalidExpertsToSentinel) { + expert_id = sentinel_expert_bucket(topk_ids[i], num_experts); + } else { + expert_id = topk_ids[i] + 1; + } int32_t rank_post_pad = tokens_cnts[tid * num_experts + expert_id] + cumsum[expert_id]; sorted_token_ids[rank_post_pad] = i; ++tokens_cnts[tid * num_experts + expert_id]; @@ -338,7 +398,12 @@ __global__ void moe_align_block_size_small_batch_expert_kernel( // With 1024 threads and EXPERTS_PER_THREAD=4, covers at most 4096 expert // indices. Since num_experts includes the +1 offset bucket, this supports // up to 4095 real experts. -template +template < + typename scalar_t, + int EXPERTS_PER_THREAD, + bool kWriteInactivePrefixCount, + bool kMapInvalidExpertsToSentinel, + bool kPadExpertIds> __global__ void moe_align_block_size_kernel_v2( const scalar_t* __restrict__ topk_ids, int32_t* __restrict__ sorted_token_ids, @@ -388,9 +453,14 @@ __global__ void moe_align_block_size_kernel_v2( __syncthreads(); for (size_t i = tid; i < numel; i += blockDim.x) { - int expert_id = topk_ids[i] + 1; // +1 offset convention - if (expert_id < num_experts) { + if constexpr (kMapInvalidExpertsToSentinel) { + const int32_t expert_id = sentinel_expert_bucket(topk_ids[i], num_experts); atomicAdd(&shared_counts[expert_id], 1); + } else { + int expert_id = topk_ids[i] + 1; // +1 offset convention + if (expert_id < num_experts) { + atomicAdd(&shared_counts[expert_id], 1); + } } } @@ -439,7 +509,12 @@ __global__ void moe_align_block_size_kernel_v2( // Last thread writes total if (tid == blockDim.x - 1) { cumsum[num_experts] = thread_prefix + thread_sum; - *total_tokens_post_pad = thread_prefix + thread_sum; + if constexpr (kWriteInactivePrefixCount) { + *reinterpret_cast(total_tokens_post_pad) = + make_int2(thread_prefix + thread_sum, CEILDIV(shared_counts[0], block_size)); + } else { + *total_tokens_post_pad = thread_prefix + thread_sum; + } } __syncthreads(); @@ -462,7 +537,11 @@ __global__ void moe_align_block_size_kernel_v2( } // namespace moe -template +template < + typename scalar_t, + bool kWriteInactivePrefixCount = false, + bool kMapInvalidExpertsToSentinel = false, + bool kPadExpertIds = false> struct MoeAlignBlockSizeKernel { static void run(tvm::ffi::TensorView topk_ids, @@ -476,6 +555,9 @@ struct MoeAlignBlockSizeKernel { using namespace host; auto device = topk_ids.device(); + if constexpr (kWriteInactivePrefixCount) { + TensorMatcher({2}).with_dtype().with_device(device).ensure_alignment(8).verify(num_tokens_post_pad); + } const cudaStream_t stream = LaunchKernel::resolve_device(device); int threads = 1024; @@ -488,6 +570,10 @@ struct MoeAlignBlockSizeKernel { // to 8, covering at most 8192 expert indices. This supports up to 8191 real // experts, sufficient for LoRA virtual experts (num_moe_experts * max_loras). RuntimeCheck(num_experts <= 8192, "moe_align_block_size: num_experts must be <= 8192, got ", num_experts); + if constexpr (kMapInvalidExpertsToSentinel) { + RuntimeCheck( + num_experts > 0 && block_size > 0, "moe_align_block_size: bucket count and block size must be positive"); + } const scalar_t* topk_ids_ptr = static_cast(topk_ids.data_ptr()); int32_t* sorted_token_ids_ptr = static_cast(sorted_token_ids.data_ptr()); @@ -503,7 +589,12 @@ struct MoeAlignBlockSizeKernel { constexpr int32_t fill_threads = 256; const int32_t shared_mem_size = ((num_thread + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t); - auto kernel = moe::moe_align_block_size_small_batch_expert_kernel; + auto kernel = moe::moe_align_block_size_small_batch_expert_kernel< + scalar_t, + fill_threads, + kWriteInactivePrefixCount, + kMapInvalidExpertsToSentinel, + kPadExpertIds>; LaunchKernel(dim3(1), dim3(fill_threads + num_thread), stream, shared_mem_size)( kernel, topk_ids_ptr, @@ -520,7 +611,8 @@ struct MoeAlignBlockSizeKernel { const size_t scan_size = host::round_up_pow2(static_cast(num_experts)); const size_t shared_mem_size = (num_experts + (num_experts + 1) + scan_size + WARP_SIZE) * sizeof(int32_t); - auto align_kernel = moe::moe_align_block_size_kernel; + auto align_kernel = moe:: + moe_align_block_size_kernel; LaunchKernel(dim3(2), dim3(threads), stream, shared_mem_size)( align_kernel, topk_ids_ptr, @@ -541,9 +633,22 @@ struct MoeAlignBlockSizeKernel { const int max_blocks = 65535; const int actual_blocks = std::min(num_blocks, max_blocks); - auto sort_kernel = moe::count_and_sort_expert_tokens_kernel; - LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)( - sort_kernel, topk_ids_ptr, sorted_token_ids_ptr, cumsum_buffer_ptr, numel); + if (actual_blocks > 0) { + if constexpr (kMapInvalidExpertsToSentinel) { + auto sort_kernel = moe::count_and_sort_checked_expert_tokens_kernel; + LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)( + sort_kernel, + topk_ids_ptr, + sorted_token_ids_ptr, + cumsum_buffer_ptr, + numel, + static_cast(num_experts)); + } else { + auto sort_kernel = moe::count_and_sort_expert_tokens_kernel; + LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)( + sort_kernel, topk_ids_ptr, sorted_token_ids_ptr, cumsum_buffer_ptr, numel); + } + } } else { // v2 path for >1024 experts: two-level warp scan with EXPERTS_PER_THREAD int64_t padded_num_experts = ((num_experts + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE; @@ -551,7 +656,12 @@ struct MoeAlignBlockSizeKernel { auto launch_v2 = [&](auto ept_tag) { constexpr int EPT = decltype(ept_tag)::value; - auto v2_kernel = moe::moe_align_block_size_kernel_v2; + auto v2_kernel = moe::moe_align_block_size_kernel_v2< + scalar_t, + EPT, + kWriteInactivePrefixCount, + kMapInvalidExpertsToSentinel, + kPadExpertIds>; LaunchKernel(dim3(2), dim3(threads), stream, shared_mem_size)( v2_kernel, topk_ids_ptr, @@ -581,9 +691,22 @@ struct MoeAlignBlockSizeKernel { const int max_blocks = 65535; const int actual_blocks = std::min(num_blocks, max_blocks); - auto sort_kernel = moe::count_and_sort_expert_tokens_kernel; - LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)( - sort_kernel, topk_ids_ptr, sorted_token_ids_ptr, cumsum_buffer_ptr, numel); + if (actual_blocks > 0) { + if constexpr (kMapInvalidExpertsToSentinel) { + auto sort_kernel = moe::count_and_sort_checked_expert_tokens_kernel; + LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)( + sort_kernel, + topk_ids_ptr, + sorted_token_ids_ptr, + cumsum_buffer_ptr, + numel, + static_cast(num_experts)); + } else { + auto sort_kernel = moe::count_and_sort_expert_tokens_kernel; + LaunchKernel(dim3(actual_blocks), dim3(block_threads), stream)( + sort_kernel, topk_ids_ptr, sorted_token_ids_ptr, cumsum_buffer_ptr, numel); + } + } } } }; diff --git a/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/topk_softmax_pack.py b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/topk_softmax_pack.py index 550f6bcb92..a2b710381f 100644 --- a/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/topk_softmax_pack.py +++ b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/topk_softmax_pack.py @@ -71,6 +71,9 @@ def topk_softmax_pack( pack id = -1 (the padded-region sentinel). ``topk_weights``/``topk_indices`` are written exactly like the AOT kernel (indices NOT masked here — the regular python post-process handles them). + + The real-row count may be a scalar or a one-element vector. A scalar is + viewed as the vector expected by the native launcher without copying it. """ assert gating_output.dim() == 2 num_experts = gating_output.shape[-1] @@ -80,6 +83,8 @@ def topk_softmax_pack( ) if gating_output.shape[0] == 0: return + if num_token_non_padded is not None and num_token_non_padded.dim() == 0: + num_token_non_padded = num_token_non_padded.view(1) _jit_topk_softmax_pack_op( topk_weights, topk_indices, diff --git a/python/sglang/kernels/ops/lora/moe/virtual_experts.py b/python/sglang/kernels/ops/lora/moe/virtual_experts.py index 6267cfe217..cd194e77b7 100644 --- a/python/sglang/kernels/ops/lora/moe/virtual_experts.py +++ b/python/sglang/kernels/ops/lora/moe/virtual_experts.py @@ -307,6 +307,9 @@ def _align_block_size_jit( block_size: int, num_experts: int, scratch=None, + *, + return_inactive_prefix_count: bool = False, + map_invalid_experts_to_sentinel: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """CUDA JIT align_block_size for num_experts > 1024 (up to 8191). @@ -316,6 +319,8 @@ def _align_block_size_jit( ``scratch(numel)`` returns the int32 scratch buffer (default: a fresh allocation); a caller under CUDA graphs passes a persistent buffer. + IDs must be certified in [-1, num_experts) unless + ``map_invalid_experts_to_sentinel`` maps other signed IDs to bucket zero. """ assert num_experts <= 8191, ( f"_align_block_size_jit supports at most 8191 experts " @@ -324,13 +329,21 @@ def _align_block_size_jit( device = topk_ids.device flat_topk_ids = topk_ids.reshape(-1) - if flat_topk_ids.dtype == torch.int64: + if not map_invalid_experts_to_sentinel and flat_topk_ids.dtype == torch.int64: flat_topk_ids = flat_topk_ids.to(torch.int32) num_total_tokens = flat_topk_ids.numel() if num_total_tokens == 0: empty = torch.empty(0, dtype=torch.int32, device=device) - return empty, empty, torch.zeros(1, dtype=torch.int32, device=device) + return ( + empty, + empty, + torch.zeros( + 2 if return_inactive_prefix_count else 1, + dtype=torch.int32, + device=device, + ), + ) # JIT kernel uses +1 offset convention: -1 -> bucket 0 (sentinel), # expert i -> bucket i+1. So pass num_experts + 1 as the bucket count. @@ -370,7 +383,9 @@ def _align_block_size_jit( off += max_num_tokens_padded expert_ids = buf[off : off + max_num_m_blocks] off += max_num_m_blocks_padded - num_tokens_post_padded = buf[off : off + 1] + # The already padded count segment has room for the optional pair: + # [total padded tokens, leading negative-expert blocks]. + num_tokens_post_padded = buf[off : off + (2 if return_inactive_prefix_count else 1)] off += num_post_pad_size cumsum_buffer = buf[off : off + jit_num_experts + 1] @@ -383,6 +398,8 @@ def _align_block_size_jit( num_tokens_post_padded, cumsum_buffer, True, # pad_sorted_token_ids + write_inactive_prefix_count=return_inactive_prefix_count, + map_invalid_experts_to_sentinel=map_invalid_experts_to_sentinel, ) return sorted_token_ids, expert_ids, num_tokens_post_padded diff --git a/python/sglang/kernels/ops/moe/moe_align.py b/python/sglang/kernels/ops/moe/moe_align.py index 9dd8460151..52cb127a0e 100644 --- a/python/sglang/kernels/ops/moe/moe_align.py +++ b/python/sglang/kernels/ops/moe/moe_align.py @@ -11,8 +11,20 @@ if TYPE_CHECKING: @cache_once -def _jit_moe_align_module(dtype: torch.dtype, pad_expert_ids: bool) -> Module: - args = make_cpp_args(dtype, pad_expert_ids) +def _jit_moe_align_module( + dtype: torch.dtype, + write_inactive_prefix_count: bool = False, + map_invalid_experts_to_sentinel: bool = False, + pad_expert_ids: bool = False, +) -> Module: + if map_invalid_experts_to_sentinel and dtype not in (torch.int32, torch.int64): + raise ValueError("Checked alignment requires int32 or int64 expert IDs") + args = make_cpp_args( + dtype, + write_inactive_prefix_count, + map_invalid_experts_to_sentinel, + pad_expert_ids, + ) return load_jit( "moe_align_block_size", *args, @@ -33,10 +45,23 @@ def moe_align_block_size( cumsum_buffer: torch.Tensor, pad_sorted_token_ids: bool = False, *, + write_inactive_prefix_count: bool = False, + map_invalid_experts_to_sentinel: bool = False, pad_expert_ids: bool = False, ) -> None: - """Align routes; optionally define unused expert-map capacity as -1.""" - module = _jit_moe_align_module(topk_ids.dtype, pad_expert_ids) + """Align +1-offset buckets, optionally publishing a padded-token/block pair. + + ``write_inactive_prefix_count`` requires an aligned int32[2] count output. + ``pad_expert_ids`` fills unused expert-map capacity with -1. + IDs must be in ``[-1, num_experts - 2]`` unless + ``map_invalid_experts_to_sentinel`` maps other signed IDs into bucket zero. + """ + module = _jit_moe_align_module( + topk_ids.dtype, + write_inactive_prefix_count, + map_invalid_experts_to_sentinel, + pad_expert_ids, + ) module.moe_align_block_size( topk_ids, num_experts, diff --git a/python/sglang/kernels/ops/moe/moe_align_single_token.py b/python/sglang/kernels/ops/moe/moe_align_single_token.py index 68677f3254..27b7c4dadd 100644 --- a/python/sglang/kernels/ops/moe/moe_align_single_token.py +++ b/python/sglang/kernels/ops/moe/moe_align_single_token.py @@ -18,8 +18,8 @@ if TYPE_CHECKING: @cache_once -def _jit_align_single_token_module() -> Module: - args = make_cpp_args(is_arch_support_pdl()) +def _jit_align_single_token_module(write_inactive_prefix_count: bool = False) -> Module: + args = make_cpp_args(is_arch_support_pdl(), write_inactive_prefix_count) return load_jit( "moe_align_single_token", *args, @@ -30,20 +30,26 @@ def _jit_align_single_token_module() -> Module: def moe_align_single_token( - topk_ids: torch.Tensor, block_size: int + topk_ids: torch.Tensor, + block_size: int, + *, + return_inactive_prefix_count: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """moe_align_block_size for a single token (distinct expert ids). Returns (sorted_token_ids, expert_ids, num_tokens_post_padded) with the same layout as moe_align_block_size: experts ascending, one block per - expert, padding value = numel. + expert, padding value = numel. With ``return_inactive_prefix_count``, + the count is int32 ``[padded_tokens, inactive_blocks]``. """ topk = topk_ids.shape[1] device = topk_ids.device sorted_ids = torch.empty((topk * block_size,), dtype=torch.int32, device=device) expert_ids = torch.empty((topk,), dtype=torch.int32, device=device) - num_post = torch.empty((1,), dtype=torch.int32, device=device) - _jit_align_single_token_module().run( + num_post = torch.empty( + (2 if return_inactive_prefix_count else 1,), dtype=torch.int32, device=device + ) + _jit_align_single_token_module(return_inactive_prefix_count).run( topk_ids, sorted_ids, expert_ids, num_post, block_size ) return sorted_ids, expert_ids, num_post diff --git a/python/sglang/kernels/ops/moe/moe_align_small_numel.py b/python/sglang/kernels/ops/moe/moe_align_small_numel.py index add436ede5..c1a2d5f909 100644 --- a/python/sglang/kernels/ops/moe/moe_align_small_numel.py +++ b/python/sglang/kernels/ops/moe/moe_align_small_numel.py @@ -24,13 +24,14 @@ def _moe_align_small_numel_kernel( topk_ids_ptr, # [numel] int, flattened (token, slot) expert ids, -1 = filtered sorted_token_ids_ptr, # [max_num_tokens_padded] int32 expert_ids_ptr, # [max_num_m_blocks] int32 - num_tokens_post_pad_ptr, # [1] int32 + num_tokens_post_pad_ptr, # [1] or [2] int32 num_experts, # E + 1 (the "+1 offset" convention's bucket count) block_size, numel, NP: tl.constexpr, # power-of-2 >= numel NB: tl.constexpr, # power-of-2 >= max blocks used USE_GDC: tl.constexpr = False, + WRITE_INACTIVE_PREFIX_COUNT: tl.constexpr = False, ): """Single-CTA moe_align for tiny batches with MANY experts. @@ -77,6 +78,11 @@ def _moe_align_small_numel_kernel( total = tl.sum(tl.where(is_rep, padded_cnt, 0), axis=0) tl.store(num_tokens_post_pad_ptr, total.to(tl.int32)) + if WRITE_INACTIVE_PREFIX_COUNT: + inactive_blocks = tl.sum( + tl.where(is_rep & (bucket == 0), padded_cnt // block_size, 0), axis=0 + ) + tl.store(num_tokens_post_pad_ptr + 1, inactive_blocks.to(tl.int32)) # expert_ids per used block: representative r owns blocks # [excl[r], excl[r] + padded_cnt[r]); the written id is bucket - 1 @@ -118,6 +124,8 @@ def moe_align_small_numel( sorted_token_ids: torch.Tensor, expert_ids: torch.Tensor, num_tokens_post_pad: torch.Tensor, + *, + write_inactive_prefix_count: bool = False, ) -> None: """Align and sort expert token ids into block-padded buffers, in one launch. @@ -130,6 +138,12 @@ def moe_align_small_numel( correct above it, but its pairwise tensors spill and it stops being faster than the two-kernel path. """ + if write_inactive_prefix_count: + assert num_tokens_post_pad.shape == (2,) + assert num_tokens_post_pad.dtype == torch.int32 + assert num_tokens_post_pad.device == topk_ids.device + assert num_tokens_post_pad.is_contiguous() + assert num_tokens_post_pad.data_ptr() % 8 == 0 numel = topk_ids.numel() pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {} _moe_align_small_numel_kernel[(1,)]( @@ -142,6 +156,7 @@ def moe_align_small_numel( numel, NP=triton.next_power_of_2(max(numel, 2)), NB=triton.next_power_of_2(max(expert_ids.numel(), 2)), + WRITE_INACTIVE_PREFIX_COUNT=write_inactive_prefix_count, num_warps=4, **pdl_kwargs, ) diff --git a/python/sglang/kernels/ops/moe/moe_wna16_marlin.py b/python/sglang/kernels/ops/moe/moe_wna16_marlin.py index e5d9d6a6f6..f5d7d2ce34 100644 --- a/python/sglang/kernels/ops/moe/moe_wna16_marlin.py +++ b/python/sglang/kernels/ops/moe/moe_wna16_marlin.py @@ -17,9 +17,21 @@ _MAX_THREAD_N = 256 @cache_once def _jit_moe_wna16_marlin_module( - dtype: torch.dtype, is_ep: bool, has_bias: bool + dtype: torch.dtype, + is_ep: bool, + has_bias: bool, + expert_ids_are_sorted: bool = False, + zero_inactive_output: bool = False, + has_inactive_prefix_count: bool = False, ) -> Module: - args = make_cpp_args(dtype, is_ep, has_bias) + args = make_cpp_args( + dtype, + is_ep, + has_bias, + expert_ids_are_sorted, + zero_inactive_output, + has_inactive_prefix_count, + ) return load_jit( "moe_wna16_marlin", *args, @@ -67,10 +79,22 @@ def moe_wna16_marlin_gemm( use_atomic_add: bool = False, use_fp32_reduce: bool = False, is_zp_float: bool = False, + expert_ids_are_sorted: bool = False, + zero_inactive_output: bool = False, + has_inactive_prefix_count: bool = False, ) -> torch.Tensor: """Project routes, preserving inactive rows in a caller-provided output. - Inactive rows are zero when this function allocates the output. + Inactive rows are zero when this function allocates the output. With + ``zero_inactive_output=True``, the GEMM also zeros caller-owned rows present + in inactive routing blocks; omitted routes remain untouched. + ``expert_ids_are_sorted`` certifies ascending block expert IDs within the + padded token count, with any -1 blocks first. Unused tail entries are ignored. + Leave it false after remapping that can produce scattered inactive blocks. + The sorted specialization is only used without expert parallelism. + ``has_inactive_prefix_count`` additionally certifies that a supporting + producer wrote an aligned int32 pair ``[padded_tokens, inactive_blocks]`` + for these tables and this block size. It is ignored outside sorted non-EP. """ device = a.device @@ -140,7 +164,14 @@ def moe_wna16_marlin_gemm( b_bias_t = _or_empty(b_bias_or_none, device, a.dtype) global_scale_t = _or_empty(global_scale_or_none, device, a.dtype) - module = _jit_moe_wna16_marlin_module(a.dtype, is_ep, has_bias) + module = _jit_moe_wna16_marlin_module( + a.dtype, + is_ep, + has_bias, + expert_ids_are_sorted and not is_ep, + zero_inactive_output, + has_inactive_prefix_count and expert_ids_are_sorted and not is_ep, + ) module.moe_wna16_marlin_gemm( a, c, diff --git a/python/sglang/kernels/ops/moe/muse_spark_post_expert_finalize_triton.py b/python/sglang/kernels/ops/moe/muse_spark_post_expert_finalize_triton.py index b2092f6e2a..ed451fd01b 100644 --- a/python/sglang/kernels/ops/moe/muse_spark_post_expert_finalize_triton.py +++ b/python/sglang/kernels/ops/moe/muse_spark_post_expert_finalize_triton.py @@ -21,6 +21,7 @@ def _post_expert_norm_weighted_sum_kernel( HAS_WEIGHT: tl.constexpr, WEIGHT_FIRST: tl.constexpr, HAS_IDS: tl.constexpr, + ROUND_PRODUCT: tl.constexpr, BLOCK_H: tl.constexpr, ): tok = tl.program_id(0) @@ -55,7 +56,10 @@ def _post_expert_norm_weighted_sum_kernel( y = x s = tl.load(s_ptr + tok * TOPK + k).to(tl.float32) - acc += y * s + product = y * s + if ROUND_PRODUCT: + product = product.to(x_ptr.dtype.element_ty).to(tl.float32) + acc += product tl.store(out_ptr + tok * H + offs, acc.to(out_ptr.dtype.element_ty), mask=mask) @@ -68,20 +72,20 @@ def fused_post_expert_norm_weighted_sum( weight_first: bool = True, apply_norm: bool = True, slot_expert_ids: Optional[torch.Tensor] = None, + round_product: bool = False, ) -> torch.Tensor: + """Normalize and combine experts without reading inactive slots. + + round_product preserves input-dtype multiplication before FP32 accumulation + when combining an already normalized low-precision tensor. + """ x = expert_output.contiguous() tokens, topk, hidden = x.shape out = torch.empty((tokens, hidden), dtype=x.dtype, device=x.device) if tokens == 0: return out - if tokens < _MIN_TOKENS_FOR_FUSED: + if tokens < _MIN_TOKENS_FOR_FUSED and slot_expert_ids is None and not round_product: y = x - if slot_expert_ids is not None: - y = torch.where( - (slot_expert_ids.reshape(tokens, topk) >= 0).unsqueeze(-1), - y, - torch.zeros((), dtype=y.dtype, device=y.device), - ) if apply_norm: f = y.float() if norm_weight is not None and weight_first: @@ -101,10 +105,10 @@ def fused_post_expert_norm_weighted_sum( block_h = triton.next_power_of_2(hidden) _post_expert_norm_weighted_sum_kernel[(tokens,)]( x, - norm_weight if norm_weight is not None else x, + norm_weight.contiguous() if norm_weight is not None else x, topk_scores.contiguous(), out, - slot_expert_ids if slot_expert_ids is not None else x, + slot_expert_ids.contiguous() if slot_expert_ids is not None else x, TOPK=topk, H=hidden, eps=eps, @@ -112,6 +116,7 @@ def fused_post_expert_norm_weighted_sum( HAS_WEIGHT=norm_weight is not None, WEIGHT_FIRST=weight_first, HAS_IDS=slot_expert_ids is not None, + ROUND_PRODUCT=round_product, BLOCK_H=block_h, num_warps=8 if block_h >= 4096 else 4, num_stages=2, diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py b/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py index 53ca979a39..1662b50447 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py @@ -275,11 +275,14 @@ def fused_marlin_moe( from sglang.kernels.ops.moe.moe_align_single_token import moe_align_single_token sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_single_token( - topk_ids, block_size_m + topk_ids, block_size_m, return_inactive_prefix_count=True ) else: sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( - topk_ids, block_size_m, global_num_experts + topk_ids, + block_size_m, + global_num_experts, + return_inactive_prefix_count=expert_map is None, ) if workspace is None: @@ -339,6 +342,8 @@ def fused_marlin_moe( top_k=topk, mul_topk_weights=False, is_ep=expert_map is not None, + expert_ids_are_sorted=expert_map is None, + has_inactive_prefix_count=expert_map is None, b_q_type=scalar_type1, size_m=M, size_n=gemm1_n, @@ -380,9 +385,7 @@ def fused_marlin_moe( else: raise ValueError(f"Unsupported activation: {activation=}, with {is_gated=}") - # Gate-up is dead after activation, so its storage can hold the down output. - # Skipped down routes must start at zero before the top-k reduction. - intermediate_cache3.zero_() + # Gate-up is dead; the down GEMM defines inactive rows in the shared output. intermediate_cache3 = moe_wna16_marlin_gemm( intermediate_cache2, intermediate_cache3, @@ -401,7 +404,10 @@ def fused_marlin_moe( moe_block_size=block_size_m, top_k=1, mul_topk_weights=True, + zero_inactive_output=True, is_ep=expert_map is not None, + expert_ids_are_sorted=expert_map is None, + has_inactive_prefix_count=expert_map is None, b_q_type=scalar_type2, size_m=M * topk, size_n=K, diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py b/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py index 0b63a3a909..1c7b31b054 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py @@ -35,6 +35,7 @@ def moe_align_block_size( num_experts: int, ignore_invalid_expert: bool = False, *, + return_inactive_prefix_count: bool = False, pad_expert_ids: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ @@ -52,7 +53,9 @@ def moe_align_block_size( to their allocated expert. - expert_ids: A tensor indicating the assigned expert index for each block. - num_tokens_post_padded: The total number of tokens after padding, - ensuring divisibility by block_size. + ensuring divisibility by block_size. Opting into ``return_inactive_prefix_count`` + returns int32 ``[padded_tokens, inactive_blocks]`` from a supporting local + producer; the second word counts negative physical blocks, not tokens. This function pads the number of tokens that each expert needs to process so that it is divisible by block_size. @@ -79,6 +82,11 @@ def moe_align_block_size( "Padding expert IDs requires CUDA and the invalid-expert bucket" ) + if return_inactive_prefix_count: + if not _is_cuda or ignore_invalid_expert or num_experts > 8191: + raise ValueError( + "Inactive-prefix counts require CUDA alignment with at most 8191 experts and the +1 bucket convention" + ) # ===== TO BE REFACTORED ==== if _SGLANG_EXPERIMENTAL_LORA_OPTI and not pad_expert_ids: from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs @@ -88,7 +96,20 @@ def moe_align_block_size( _align_block_size_jit, ) - return _align_block_size_jit(topk_ids, block_size, num_experts) + if return_inactive_prefix_count: + return _align_block_size_jit( + topk_ids, + block_size, + num_experts, + return_inactive_prefix_count=return_inactive_prefix_count, + map_invalid_experts_to_sentinel=False, + ) + return _align_block_size_jit( + topk_ids, + block_size, + num_experts, + map_invalid_experts_to_sentinel=False, + ) # ===== END TO BE REFACTORED ==== if topk_ids.numel() < num_experts + 1: @@ -105,7 +126,11 @@ def moe_align_block_size( expert_ids = torch.empty( (max_num_m_blocks,), dtype=torch.int32, device=topk_ids.device ) - num_tokens_post_pad = torch.empty((1), dtype=torch.int32, device=topk_ids.device) + num_tokens_post_pad = torch.empty( + (2 if return_inactive_prefix_count else 1,), + dtype=torch.int32, + device=topk_ids.device, + ) # In EP, expert_ids for filtered experts are -1. We have num_experts + 1 ids in total. cumsum_buffer = torch.empty( @@ -132,15 +157,18 @@ def moe_align_block_size( sorted_ids, expert_ids, num_tokens_post_pad, + write_inactive_prefix_count=return_inactive_prefix_count, ) return sorted_ids, expert_ids, num_tokens_post_pad # ===== TO BE REFACTORED ==== - use_jit_align = pad_expert_ids + use_jit_align = return_inactive_prefix_count or pad_expert_ids if _SGLANG_EXPERIMENTAL_LORA_OPTI: from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs - use_jit_align |= lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get() + use_jit_align = ( + use_jit_align or lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get() + ) if use_jit_align: from sglang.kernels.ops.moe.moe_align import ( moe_align_block_size as jit_moe_align_block_size, @@ -155,6 +183,7 @@ def moe_align_block_size( num_tokens_post_pad, cumsum_buffer, True, + write_inactive_prefix_count=return_inactive_prefix_count, pad_expert_ids=pad_expert_ids, ) # ===== END TO BE REFACTORED ==== diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py index e697d07c9d..41c463dcfa 100644 --- a/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py +++ b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py @@ -382,7 +382,8 @@ class MuseSpark1xFp4Experts(nn.Module): hidden_states: torch.Tensor, topk_scores: torch.Tensor, topk_ids: torch.Tensor, - ) -> torch.Tensor: + ) -> tuple[torch.Tensor, torch.Tensor]: + """Project local routes; finalization skips undefined inactive slots.""" from sgl_kernel.scalar_type import scalar_types from sglang.kernels.ops.activation.activation import silu_and_mul @@ -416,7 +417,9 @@ class MuseSpark1xFp4Experts(nn.Module): ) sorted_token_ids, expert_ids, num_tokens_post_padded = ( - moe_align_single_token(local_topk_ids, block_size_m) + moe_align_single_token( + local_topk_ids, block_size_m, return_inactive_prefix_count=True + ) ) else: align_num_experts = ( @@ -425,22 +428,27 @@ class MuseSpark1xFp4Experts(nn.Module): else self.num_local_experts ) sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( - local_topk_ids, block_size_m, align_num_experts + local_topk_ids, + block_size_m, + align_num_experts, + return_inactive_prefix_count=expert_map is None, ) + # Both aligners emit ascending block IDs with inactive blocks first; + # no expert remapping occurs after this ordering is established. workspace = marlin_make_workspace(hidden_states.device, max_blocks_per_sm=4) cache2 = torch.empty( (num_tokens * topk, size_n), device=hidden_states.device, dtype=hidden_states.dtype, ) - cache13 = torch.zeros( - (num_tokens * topk * (gemm1_n + size_k),), + cache13 = torch.empty( + (num_tokens * topk * max(gemm1_n, size_k),), device=hidden_states.device, dtype=hidden_states.dtype, ) cache1 = cache13[: num_tokens * topk * gemm1_n].view(-1, gemm1_n) - cache3 = cache13[num_tokens * topk * gemm1_n :].view(-1, size_k) + cache3 = cache13[: num_tokens * topk * size_k].view(-1, size_k) cache1 = moe_wna16_marlin_gemm( hidden_states, @@ -469,6 +477,8 @@ class MuseSpark1xFp4Experts(nn.Module): use_atomic_add=False, use_fp32_reduce=True, is_zp_float=False, + expert_ids_are_sorted=True, + has_inactive_prefix_count=expert_map is None, ) silu_and_mul( cache1.view(-1, gemm1_n), @@ -503,6 +513,8 @@ class MuseSpark1xFp4Experts(nn.Module): use_atomic_add=False, use_fp32_reduce=True, is_zp_float=False, + expert_ids_are_sorted=True, + has_inactive_prefix_count=expert_map is None, ).view(num_tokens, topk, size_k), local_topk_ids def _forward_marlin( @@ -529,7 +541,6 @@ class MuseSpark1xFp4Experts(nn.Module): expert_outputs, local_topk_ids = self._marlin_expert_outputs( hidden_states, topk_scores, topk_ids ) - topk_scores = topk_scores.masked_fill(local_topk_ids < 0, 0.0) if self._post_expert_norm is not post_expert_norm: self.set_post_expert_norm(post_expert_norm) if self._post_expert_norm_weight is not None: @@ -546,6 +557,7 @@ class MuseSpark1xFp4Experts(nn.Module): topk_scores=topk_scores, post_expert_norm=post_expert_norm, out_dtype=hidden_states.dtype, + slot_expert_ids=local_topk_ids, ) def prepare_input( @@ -756,10 +768,20 @@ def finalize_identity_with_post_expert_norm( topk_scores: torch.Tensor, post_expert_norm: nn.Module, out_dtype: torch.dtype, + slot_expert_ids: Optional[torch.Tensor] = None, ) -> torch.Tensor: + """Apply row-local expert normalization before combining live routes.""" num_tokens, top_k = expert_outputs.shape[0], expert_outputs.shape[1] normed = post_expert_norm(expert_outputs) weights = topk_scores.reshape(num_tokens, top_k).to(normed.dtype) + if slot_expert_ids is not None: + return fused_post_expert_norm_weighted_sum( + normed, + weights, + apply_norm=False, + slot_expert_ids=slot_expert_ids, + round_product=True, + ).to(out_dtype) return (normed * weights.unsqueeze(-1)).sum(dim=1).to(out_dtype) diff --git a/test/registered/kernels/ops/moe/test_marlin_packed_topk_unpack.py b/test/registered/kernels/ops/moe/test_marlin_packed_topk_unpack.py index e0e541eee3..b5736171db 100644 --- a/test/registered/kernels/ops/moe/test_marlin_packed_topk_unpack.py +++ b/test/registered/kernels/ops/moe/test_marlin_packed_topk_unpack.py @@ -10,12 +10,14 @@ distributions. """ import sys +from unittest.mock import patch import pytest import torch from sglang.srt.layers.moe.moe_runner.marlin import _fused_unpack_packed_topk from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") @@ -86,5 +88,84 @@ def test_unpack_empty(): assert ids.dtype == torch.int32 and w.dtype == torch.float32 +class TestPackedTopKRowCount(CustomTestCase): + def test_scalar_and_vector_count_share_storage_and_replay_mask(self): + """A captured packed router reads scalar/vector live counts as they change.""" + from sglang.kernels.ops.lora.moe.trtllm_lora_temp import topk_softmax_pack + + num_tokens, num_experts, top_k = 7, 8, 2 + scores = torch.arange(num_experts, device="cuda", dtype=torch.float32) + scores = scores.repeat(num_tokens, 1) * 0.125 + probs = scores.softmax(dim=-1) + ref_weights, ref_ids = probs.topk(top_k, dim=-1) + real_op = topk_softmax_pack._jit_topk_softmax_pack_op + for shape in ((), (1,)): + for renormalize in (False, True): + with self.subTest(shape=shape, renormalize=renormalize): + count = torch.full(shape, 3, dtype=torch.int32, device="cuda") + weights = torch.empty( + (num_tokens, top_k), dtype=torch.float32, device="cuda" + ) + ids = torch.empty_like(weights, dtype=torch.int32) + packed = torch.empty_like(ids) + observed = [] + + def observed_op(*args): + bound = args[4] + result = real_op(*args) + self.assertEqual(bound.shape, (1,)) + self.assertEqual(bound.data_ptr(), count.data_ptr()) + self.assertEqual( + bound.untyped_storage().data_ptr(), + count.untyped_storage().data_ptr(), + ) + if shape == (1,): + self.assertIs(bound, count) + observed.append(bound.data_ptr()) + return result + + def run(): + topk_softmax_pack.topk_softmax_pack( + weights, + ids, + packed, + scores, + renormalize=renormalize, + num_token_non_padded=count, + ) + + def check(live): + expected_weights = ref_weights + if renormalize: + expected_weights = ref_weights / ref_weights.sum( + dim=-1, keepdim=True + ) + torch.testing.assert_close( + weights, expected_weights, rtol=1e-5, atol=1e-7 + ) + self.assertTrue(torch.equal(ids, ref_ids.to(torch.int32))) + expected_ids = ref_ids.to(torch.int32).clone() + expected_ids[live:] = -1 + self.assertTrue( + torch.equal(packed, _torch_pack(expected_ids, weights)) + ) + self.assertEqual(count.item(), live) + + with patch.object( + topk_softmax_pack, "_jit_topk_softmax_pack_op", observed_op + ): + run() + check(3) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + self.assertEqual(len(observed), 2) + for live in (0, num_tokens, 3, 0): + count.fill_(live) + graph.replay() + check(live) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/test/registered/kernels/ops/moe/test_moe_align_block_size.py b/test/registered/kernels/ops/moe/test_moe_align_block_size.py index c6f83e9ea0..9d7991dc8c 100644 --- a/test/registered/kernels/ops/moe/test_moe_align_block_size.py +++ b/test/registered/kernels/ops/moe/test_moe_align_block_size.py @@ -286,9 +286,12 @@ def test_moe_align_block_size_pad_fill_stays_in_bounds( ) +@pytest.mark.parametrize("write_inactive_prefix_count", [False, True]) @pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) @pytest.mark.parametrize("num_tokens,num_experts", [(65, 4), (512, 128), (65, 1025)]) -def test_moe_align_block_size_completes_expert_capacity(dtype, num_tokens, num_experts): +def test_moe_align_block_size_completes_expert_capacity( + dtype, num_tokens, num_experts, write_inactive_prefix_count +): """Unused expert-map slots must be inactive after every graph replay.""" block_size, topk = 4, 2 original = ( @@ -310,7 +313,9 @@ def test_moe_align_block_size_completes_expert_capacity(dtype, num_tokens, num_e (blocks + 4,), num_experts, device="cuda", dtype=torch.int32 ) expert_ids = expert_storage[:blocks] - count = torch.empty(1, device="cuda", dtype=torch.int32) + count = torch.empty( + (2 if write_inactive_prefix_count else 1,), device="cuda", dtype=torch.int32 + ) cumsum = torch.empty(num_experts + 2, device="cuda", dtype=torch.int32) def align(): @@ -324,6 +329,7 @@ def test_moe_align_block_size_completes_expert_capacity(dtype, num_tokens, num_e cumsum, True, pad_expert_ids=True, + write_inactive_prefix_count=write_inactive_prefix_count, ) stream = torch.cuda.Stream() @@ -341,7 +347,9 @@ def test_moe_align_block_size_completes_expert_capacity(dtype, num_tokens, num_e sorted_ids.fill_(-99) expert_storage.fill_(num_experts) graph.replay() - used = count.item() + used = count[0].item() + if write_inactive_prefix_count: + assert count[1].item() == ceil_div((routes < 0).sum().item(), block_size) assert used % block_size == 0 assert torch.all(expert_ids[used // block_size :] == -1) assert torch.all(expert_storage[blocks:] == num_experts) @@ -363,5 +371,215 @@ def test_moe_align_block_size_completes_expert_capacity(dtype, num_tokens, num_e assert expected_start == used +@pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize( + "rows,num_experts", + [(17, 8), (512, 65), (32, 1025), (17, 129), (0, 8), (0, 129)], +) +def test_moe_align_inactive_prefix_count_graph(rows, num_experts, dtype): + """Owned count pairs describe only the used prefix, including replay changes.""" + from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import ( + moe_align_block_size as align, + ) + + block_size, topk = 8, 3 + valid = (torch.arange(rows * topk, dtype=dtype) % num_experts).view(rows, topk) + ids = valid.cuda() + + def run(): + return align( + ids, + block_size, + num_experts, + return_inactive_prefix_count=True, + ) + + run() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + sorted_ids, expert_ids, counts = run() + assert counts.shape == (2,) and counts.dtype == torch.int32 + assert counts.is_contiguous() and counts.data_ptr() % 8 == 0 + + mixed = valid.clone() + mixed.reshape(-1)[::3] = -1 + inactive = torch.full_like(valid, -1) + for routes in (mixed, inactive, valid, mixed): + ids.copy_(routes) + # Both words must be published on every replay, including empty inputs. + counts.fill_(-123) + graph.replay() + expected_groups = [] + flat = routes.reshape(-1) + for expert in sorted(set(flat.tolist())): + positions = (flat == expert).nonzero().reshape(-1).tolist() + padded = ceil_div(len(positions), block_size) * block_size + expected_groups.append((expert, positions, padded)) + expected_total = sum(padded for _, _, padded in expected_groups) + expected_prefix = sum( + padded // block_size + for expert, _, padded in expected_groups + if expert == -1 + ) + assert counts.cpu().tolist() == [expected_total, expected_prefix] + used_experts = expert_ids[: expected_total // block_size].cpu().tolist() + expected_experts = [ + expert + for expert, _, padded in expected_groups + for _ in range(padded // block_size) + ] + assert used_experts == expected_experts + used_ids = sorted_ids[:expected_total].cpu() + offset = 0 + for _, positions, padded in expected_groups: + group = used_ids[offset : offset + padded] + assert sorted(group[group < flat.numel()].tolist()) == positions + assert (group[group >= flat.numel()] == flat.numel()).all() + offset += padded + + +@pytest.mark.parametrize("route_ids", [(-1, -1, -1), (2, -1, 0), (2, 0, 1)]) +def test_moe_align_single_token_inactive_prefix_count(route_ids): + from sglang.kernels.ops.moe.moe_align_single_token import moe_align_single_token + + block_size = 8 + ids = torch.tensor([route_ids], device="cuda", dtype=torch.int32) + sorted_ids, expert_ids, counts = moe_align_single_token( + ids, block_size, return_inactive_prefix_count=True + ) + assert counts.cpu().tolist() == [len(route_ids) * block_size, route_ids.count(-1)] + assert expert_ids.cpu().tolist() == sorted(route_ids) + positions = sorted_ids.view(len(route_ids), block_size).cpu() + assert (positions[:, 1:] == len(route_ids)).all() + order = positions[:, 0].tolist() + assert sorted(order) == list(range(len(route_ids))) + assert [route_ids[i] for i in order] == sorted(route_ids) + + +@pytest.mark.parametrize("num_experts", [8, 65, 1025]) +@pytest.mark.parametrize("empty", [False, True]) +@pytest.mark.parametrize("write_inactive_prefix_count", [False, True]) +def test_moe_align_count_publication_bounds( + num_experts, empty, write_inactive_prefix_count +): + """Empty producers publish zero; legacy capacity never certifies word 1.""" + ids = torch.tensor( + [] if empty else [[-1, 0, num_experts - 1]], + device="cuda", + dtype=torch.int32, + ) + block_size = 8 + total = 0 if empty else 24 + sorted_ids = torch.empty(total, device="cuda", dtype=torch.int32) + expert_ids = torch.empty(total // block_size, device="cuda", dtype=torch.int32) + count_backing = torch.full((2,), -123, device="cuda", dtype=torch.int32) + cumsum = torch.empty(num_experts + 2, device="cuda", dtype=torch.int32) + moe_align_block_size( + ids, + num_experts + 1, + block_size, + sorted_ids, + expert_ids, + count_backing if write_inactive_prefix_count else count_backing[:1], + cumsum, + True, + write_inactive_prefix_count=write_inactive_prefix_count, + ) + prefix = (0 if empty else 1) if write_inactive_prefix_count else -123 + assert count_backing.cpu().tolist() == [total, prefix] + + +@pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize("num_experts", [8, 129, 1500]) +@pytest.mark.parametrize("write_inactive_prefix_count", [False, True]) +def test_moe_align_checked_ids_preserve_bounds_on_replay( + dtype, num_experts, write_inactive_prefix_count +): + """Invalid signed IDs occupy sentinel blocks without touching adjacent storage.""" + block_size, numel, poison = 8, 16, -100003 + valid = torch.tensor([0, num_experts - 1, 1, 0] * 4, dtype=dtype) + invalid = [-1, -3, num_experts, 2 * num_experts] + invalid += [torch.iinfo(dtype).min, torch.iinfo(dtype).max] + if dtype == torch.int64: + invalid += [-(2**40), 2**40] + mixed = valid.clone() + mixed[2 : 2 + len(invalid)] = torch.tensor(invalid, dtype=dtype) + all_invalid = torch.tensor((invalid * numel)[:numel], dtype=dtype) + ids = valid.cuda() + + def guarded(size): + backing = torch.full((size + 8,), poison, device="cuda", dtype=torch.int32) + return backing, backing[4 : 4 + size] + + max_tokens = numel * block_size + buffers = [ + guarded(max_tokens), + guarded(numel), + guarded(2 if write_inactive_prefix_count else 1), + guarded(num_experts + 2), + ] + sorted_ids, experts, counts, cumsum = [view for _, view in buffers] + + def run(routes=ids): + moe_align_block_size( + routes, + num_experts + 1, + block_size, + sorted_ids, + experts, + counts, + cumsum, + True, + write_inactive_prefix_count=write_inactive_prefix_count, + map_invalid_experts_to_sentinel=True, + ) + + run() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + for routes in (valid, mixed, all_invalid, valid): + ids.copy_(routes) + for _, view in buffers: + view.fill_(poison) + graph.replay() + torch.testing.assert_close(ids.cpu(), routes, rtol=0, atol=0) + groups = {} + for index, expert in enumerate(routes.tolist()): + expert = expert if 0 <= expert < num_experts else -1 + groups.setdefault(expert, []).append(index) + total = sum( + ceil_div(len(group), block_size) * block_size for group in groups.values() + ) + expected_counts = [total] + if write_inactive_prefix_count: + expected_counts.append(ceil_div(len(groups.get(-1, [])), block_size)) + assert counts.cpu().tolist() == expected_counts + offset, expected_experts = 0, [] + for expert, indices in sorted(groups.items()): + padded = ceil_div(len(indices), block_size) * block_size + group = sorted_ids[offset : offset + padded].cpu() + assert sorted(group[group < numel].tolist()) == indices + assert bool((group[group >= numel] == numel).all()) + expected_experts += [expert] * (padded // block_size) + offset += padded + assert experts[: total // block_size].cpu().tolist() == expected_experts + assert bool((experts[total // block_size :] == poison).all()) + for backing, _ in buffers: + assert bool((backing[:4] == poison).all()) + assert bool((backing[-4:] == poison).all()) + + for _, view in buffers: + view.fill_(poison) + run(ids[:0]) + assert counts.cpu().tolist() == ([0, 0] if write_inactive_prefix_count else [0]) + assert bool((experts == poison).all()) + for backing, _ in buffers: + assert bool((backing[:4] == poison).all()) + assert bool((backing[-4:] == poison).all()) + + if __name__ == "__main__": sys.exit(pytest.main([__file__])) diff --git a/test/registered/kernels/ops/moe/test_moe_wna16_marlin.py b/test/registered/kernels/ops/moe/test_moe_wna16_marlin.py index f7d8884ace..5cbfbae6ec 100644 --- a/test/registered/kernels/ops/moe/test_moe_wna16_marlin.py +++ b/test/registered/kernels/ops/moe/test_moe_wna16_marlin.py @@ -1,5 +1,6 @@ import itertools import sys +from functools import partial from types import SimpleNamespace from unittest.mock import patch @@ -27,7 +28,7 @@ from sglang.test.test_marlin_utils import ( marlin_quantize, ) -register_cuda_ci(est_time=720, stage="base-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=720, stage="extra-a", runner_config="1-gpu-large") def _has_aot_moe_wna16_marlin_gemm() -> bool: @@ -1121,7 +1122,21 @@ def _check_marlin_inactive_routes( activated = torch.nn.functional.silu(gate) * up projected = torch.einsum("mti,mthi->mth", activated, w2_ref[ids]) reference = (projected.float() * weights[:, :, None]).sum(1).to(x.dtype) - with patch.object(zero_copy_context, "get_moe_output", return_value=output): + projections = [] + + def checked_marlin_gemm(*args, **kwargs): + # Ordinary built-in alignment certifies both projections. + assert kwargs.get("expert_ids_are_sorted", False) + projections.append(kwargs["mul_topk_weights"]) + return moe_wna16_marlin_gemm(*args, **kwargs) + + with ( + patch.object(zero_copy_context, "get_moe_output", return_value=output), + patch( + "sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe.moe_wna16_marlin_gemm", + checked_marlin_gemm, + ), + ): for _ in range(3): run() torch.cuda.synchronize() @@ -1142,25 +1157,25 @@ def _check_marlin_inactive_routes( if hook_weights: weights, ids = hook_weights assert torch.count_nonzero(weights[ids < 0]) == 0 + if runner_kind == "fused": + assert projections == [False, True] * 4 -def test_moe_wna16_marlin_inactive_output_ownership(): - """An allocated output zeros inactive rows; caller-owned rows stay untouched.""" +@pytest.mark.parametrize("has_inactive_prefix_count", [False, True]) +def test_moe_wna16_marlin_inactive_output_ownership(has_inactive_prefix_count): + """Inactive-output policy honors sorted/EP routing and ignores unused tails.""" torch.manual_seed(57) - m, e, k, n, topk, block_m = 1, 8, 256, 256, 2, 8 - _, qweight, scales, zeros, g_idx, permutation = _marlin_inactive_weights( + m, e, k, n, topk, block_m = 1, 8, 256, 256, 3, 8 + w_ref, qweight, scales, zeros, g_idx, permutation = _marlin_inactive_weights( e, n, k, scalar_types.uint4b8 ) x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) - ids = torch.full((m, topk), -1, device="cuda", dtype=torch.int32) routing_weights = torch.ones((m, topk), device="cuda", dtype=torch.float32) - sorted_ids, expert_ids, num_post = moe_align_block_size(ids, block_m, e) workspace = torch.zeros( torch.cuda.get_device_properties(x.device).multi_processor_count * 4, device="cuda", dtype=torch.int32, ) - caller_output = torch.full((m * topk, n), 3.0, device="cuda", dtype=x.dtype) original_empty = torch.empty def poisoned_empty(*args, **kwargs): @@ -1169,36 +1184,128 @@ def test_moe_wna16_marlin_inactive_output_ownership(): tensor.fill_(float("nan")) return tensor - for output in (None, caller_output): - with patch.object(torch, "empty", side_effect=poisoned_empty): - result = _run_single_gemm( - moe_wna16_marlin_gemm, - x, - output, - qweight, - scales, - zeros, - g_idx, - permutation, - workspace, - sorted_ids, - expert_ids, - num_post, - routing_weights, - scalar_types.uint4b8, - block_m, - topk, - m, - n, - k, - False, - True, - True, - ) - expected = ( - torch.zeros_like(result) if output is None else torch.full_like(result, 3.0) + cases = ( + ("inactive-default", (-1, -1, -1), False, False, False), + ("inactive-sorted", (-1, -1, -1), True, False, False), + ("empty-sorted", (-1, -1, -1), True, False, False), + ("mixed-default", (-1, 0, 1), False, False, False), + ("mixed-sorted", (0, -1, 1), True, False, False), + ("mixed-scattered", (-1, 0, 1), False, True, False), + ("inactive-ep", (-1, -1, -1), False, False, True), + ("mixed-ep", (-1, 0, 1), False, True, True), + ("valid-default", (0, 1, 2), False, False, False), + ("valid-sorted", (0, 1, 2), True, False, False), + ) + for case, route_ids, sorted_proof, scatter, is_ep in cases: + if has_inactive_prefix_count and not sorted_proof: + continue + ids = torch.tensor([route_ids], device="cuda", dtype=torch.int32) + sorted_ids, expert_ids, num_post = moe_align_block_size( + ids, + block_m, + e, + return_inactive_prefix_count=has_inactive_prefix_count, ) - torch.testing.assert_close(result, expected, rtol=0, atol=0) + if scatter: + # Keep the matching token blocks, but move the inactive block to + # the middle so a first-ID shortcut without proof would be unsafe. + order = torch.tensor([1, 0, 2], device="cuda") + sorted_ids = sorted_ids.view(topk, block_m)[order].reshape(-1) + expert_ids = expert_ids[order] + elif case == "empty-sorted": + sorted_ids, expert_ids = sorted_ids[:0], expert_ids[:0] + num_post.zero_() + if case != "empty-sorted": + used_blocks = 1 if max(route_ids) < 0 else topk + if not scatter and route_ids[0] < 0: + # An empty inactive block guards against skipping only one; + # real routes stay unique, as the alignment producer guarantees. + empty_block = torch.full_like(sorted_ids[:block_m], topk) + sorted_ids = torch.cat( + (empty_block, sorted_ids[: used_blocks * block_m]) + ) + expert_ids = torch.cat((expert_ids[:1], expert_ids[:used_blocks])) + num_post[0].add_(block_m) + if has_inactive_prefix_count: + num_post[1].add_(1) + used_blocks += 1 + # Only the published prefix is ordered. A binary search or scan + # over allocated capacity would miscount these extra negative IDs. + dirty_ids = torch.tensor( + [0, -1, e - 1, -1], device="cuda", dtype=torch.int32 + ) + expert_ids = torch.cat((expert_ids[:used_blocks], dirty_ids)) + sorted_ids = torch.cat( + ( + sorted_ids[: used_blocks * block_m], + torch.full((4 * block_m,), topk, device="cuda", dtype=torch.int32), + ) + ) + + if not has_inactive_prefix_count and case == "valid-sorted": + # An allocated second word is not a producer certificate. The + # legacy path must ignore it and discover its own prefix. + num_post = torch.cat((num_post, torch.full_like(num_post, 0x3FFFFFFF))) + + def gemm(*args, **kwargs): + kwargs["is_ep"] = is_ep + if has_inactive_prefix_count: + kwargs["has_inactive_prefix_count"] = True + if sorted_proof: + kwargs["expert_ids_are_sorted"] = True + return moe_wna16_marlin_gemm(*args, **kwargs) + + # Omit the policy on default calls to guard caller-owned inactive rows. + for zero_inactive_output in (False, True): + project = ( + partial(gemm, zero_inactive_output=True) + if zero_inactive_output + else gemm + ) + caller_output = torch.full((m * topk, n), 3.0, device="cuda", dtype=x.dtype) + for output in (None, caller_output): + with patch.object(torch, "empty", side_effect=poisoned_empty): + result = _run_single_gemm( + project, + x, + output, + qweight, + scales, + zeros, + g_idx, + permutation, + workspace, + sorted_ids, + expert_ids, + num_post, + routing_weights, + scalar_types.uint4b8, + block_m, + topk, + m, + n, + k, + False, + True, + True, + ) + expected = ( + torch.zeros_like(result) + if output is None + else torch.full_like(result, 3.0) + ) + if zero_inactive_output and case != "empty-sorted": + expected[ids.reshape(-1) < 0] = 0 + for slot, expert in enumerate(route_ids): + if expert >= 0: + expected[slot] = (x[0].float() @ w_ref[expert].float().T).to( + expected.dtype + ) + torch.testing.assert_close(result, expected, rtol=0.03, atol=0.02) + inactive = ids.reshape(-1) < 0 + torch.testing.assert_close( + result[inactive], expected[inactive], rtol=0, atol=0 + ) def test_marlin_lora_direct_inactive_routes(): @@ -1266,13 +1373,22 @@ def test_marlin_lora_direct_inactive_routes(): assert torch.count_nonzero(down_output[ids < 0]) == 0 +@pytest.mark.parametrize("use_expert_map", [False, True]) +@pytest.mark.parametrize("weight_first", [False, True]) @pytest.mark.parametrize("fused_norm", [False, True]) -def test_muse_marlin_inactive_routes(fused_norm): - """Muse maps global routes once and masks inactive weights before either norm tail.""" +@pytest.mark.parametrize("hidden,intermediate", [(256, 256), (512, 128)]) +def test_muse_marlin_inactive_routes( + fused_norm, hidden, intermediate, weight_first, use_expert_map +): + """Muse finalization excludes undefined inactive projection rows and weights.""" from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import MuseSpark1xFp4Experts + from sglang.srt.layers.muse_spark.muse_spark_layers import ( + MuseSpark1xRMSNorm, + MuseSpark1xRMSNormWithInputScale, + ) torch.manual_seed(67) - rows, local_experts, hidden, intermediate, topk = 4, 2, 256, 256, 2 + rows, local_experts, topk = 4, 2, 2 dtype = torch.bfloat16 layer = MuseSpark1xFp4Experts.__new__(MuseSpark1xFp4Experts) torch.nn.Module.__init__(layer) @@ -1281,8 +1397,12 @@ def test_muse_marlin_inactive_routes(fused_norm): layer.params_dtype = dtype layer.intermediate_size_per_partition = intermediate layer.num_local_experts = local_experts - layer.global_num_experts = 4 - layer._expert_map = torch.tensor([-1, -1, 0, 1], device="cuda", dtype=torch.int32) + layer.global_num_experts = 4 if use_expert_map else local_experts + layer._expert_map = ( + torch.tensor([-1, -1, 0, 1], device="cuda", dtype=torch.int32) + if use_expert_map + else None + ) references = [] for prefix, n, k in ( ("w13", 2 * intermediate, hidden), @@ -1310,19 +1430,53 @@ def test_muse_marlin_inactive_routes(fused_norm): local_ids = torch.tensor( [[0, 1], [1, -1], [-1, 0], [-1, -1]], device="cuda", dtype=torch.int32 ) + if not use_expert_map: + ids = local_ids.clone() weights = torch.tensor( [[0.25, 0.75], [0.5, 0.5], [0.25, 0.75], [0.5, 0.5]], device="cuda" ) weights[local_ids < 0] = float("nan") - norm = torch.nn.RMSNorm(hidden, eps=1e-6, device="cuda", dtype=dtype) + weights[-1, 0] = float("inf") + input_weights = weights.clone() + norm_class = ( + MuseSpark1xRMSNormWithInputScale if weight_first else MuseSpark1xRMSNorm + ) + norm = norm_class(hidden, eps=1e-6).to(device="cuda", dtype=dtype) + norm.weight.data.copy_(torch.linspace(0.5, 1.5, hidden, device="cuda", dtype=dtype)) layer._post_expert_norm = norm layer._post_expert_norm_weight = norm.weight.detach() if fused_norm else None layer._post_expert_norm_eps = norm.eps - layer._post_expert_norm_weight_first = False + layer._post_expert_norm_weight_first = weight_first - expert_output, mapped = layer._marlin_expert_outputs(x, weights, ids) + projection_buffers = [] + + def record_projection_buffer(a, c, *args, **kwargs): + assert kwargs["is_ep"] is use_expert_map + assert kwargs["has_inactive_prefix_count"] is (not use_expert_map) + count = args[10] + assert count.shape == ((1,) if use_expert_map else (2,)) + assert count.dtype == torch.int32 and count.is_contiguous() + if not use_expert_map: + assert count.data_ptr() % 8 == 0 + projection_buffers.append(c) + c.fill_(float("nan")) + c[local_ids.reshape(-1) < 0, 0] = float("inf") + return moe_wna16_marlin_gemm(a, c, *args, **kwargs) + + with patch( + "sglang.kernels.ops.moe.moe_wna16_marlin.moe_wna16_marlin_gemm", + side_effect=record_projection_buffer, + ): + expert_output, mapped = layer._marlin_expert_outputs(x, weights, ids) + actual = layer._forward_marlin(x, weights, ids, norm) + for gate_up_buffer, down_buffer in zip( + projection_buffers[::2], projection_buffers[1::2] + ): + assert gate_up_buffer.data_ptr() == down_buffer.data_ptr() + assert down_buffer.untyped_storage().nbytes() == ( + rows * topk * max(2 * intermediate, hidden) * x.element_size() + ) torch.testing.assert_close(mapped, local_ids, rtol=0, atol=0) - assert torch.count_nonzero(expert_output[local_ids < 0]) == 0 expected_experts = torch.zeros((rows, topk, hidden), device="cuda", dtype=dtype) w13_ref, w2_ref = references for row in range(rows): @@ -1334,16 +1488,20 @@ def test_muse_marlin_inactive_routes(fused_norm): gate, up = gate_up.chunk(2) activated = torch.nn.functional.silu(gate) * up expected_experts[row, route] = activated @ w2_ref[expert].T - torch.testing.assert_close(expert_output, expected_experts, rtol=0.05, atol=0.25) + torch.testing.assert_close( + expert_output[local_ids >= 0], + expected_experts[local_ids >= 0], + rtol=0.05, + atol=0.25, + ) safe_weights = weights.masked_fill(local_ids < 0, 0.0) expected = ( (norm(expected_experts).float() * safe_weights[..., None]).sum(1).to(dtype) ) - actual = layer._forward_marlin(x, weights, ids, norm) torch.testing.assert_close(actual, expected, rtol=0.05, atol=0.25) assert torch.isfinite(actual).all() assert torch.count_nonzero(actual[-1]) == 0 - assert torch.isnan(weights[local_ids < 0]).all() + torch.testing.assert_close(weights, input_weights, rtol=0, atol=0, equal_nan=True) if __name__ == "__main__": diff --git a/test/registered/kernels/ops/moe/test_muse_spark_routing.py b/test/registered/kernels/ops/moe/test_muse_spark_routing.py index 29bb7f5a89..56af253460 100644 --- a/test/registered/kernels/ops/moe/test_muse_spark_routing.py +++ b/test/registered/kernels/ops/moe/test_muse_spark_routing.py @@ -76,6 +76,7 @@ def _post_expert_reference(x, scores, weight, eps, weight_first, ids): f = x.float() if ids is not None: f = torch.where((ids >= 0).unsqueeze(-1), f, torch.zeros_like(f)) + scores = torch.where(ids >= 0, scores, torch.zeros_like(scores)) w = weight.float() if weight_first: f = f * w @@ -85,7 +86,7 @@ def _post_expert_reference(x, scores, weight, eps, weight_first, ids): return (y * scores.float().unsqueeze(-1)).sum(dim=1) -@pytest.mark.parametrize("num_tokens", [5, 64]) +@pytest.mark.parametrize("num_tokens", [1, 31, 32]) @pytest.mark.parametrize("weight_first", [True, False]) @pytest.mark.parametrize("with_ids", [False, True]) def test_post_expert_finalize_triton(num_tokens, weight_first, with_ids): @@ -94,16 +95,85 @@ def test_post_expert_finalize_triton(num_tokens, weight_first, with_ids): x = torch.randn(num_tokens, topk, hidden, device="cuda", dtype=torch.bfloat16) scores = torch.rand(num_tokens, topk, device="cuda") weight = 1.0 + 0.1 * torch.randn(hidden, device="cuda", dtype=torch.bfloat16) - ids = ( - torch.randint(-1, 4, (num_tokens, topk), device="cuda", dtype=torch.int32) - if with_ids - else None - ) + ids = None + if with_ids: + ids = ( + torch.tensor([0, 1, -1, 2, -1, 3, 0, -1], device="cuda", dtype=torch.int32) + .expand(num_tokens, -1) + .clone() + ) + ids[0].fill_(-1) + poison = torch.tensor( + [float("nan"), float("inf")] * (topk // 2), device="cuda" + ).expand(num_tokens, -1) + x = torch.where((ids < 0).unsqueeze(-1), poison.unsqueeze(-1), x).to(x.dtype) + scores = torch.where(ids < 0, poison, scores) out = fused_post_expert_norm_weighted_sum( x, scores, weight, eps=1e-6, weight_first=weight_first, slot_expert_ids=ids ) ref = _post_expert_reference(x, scores, weight, 1e-6, weight_first, ids) torch.testing.assert_close(out.float(), ref, rtol=2e-2, atol=2e-2) + assert torch.isfinite(out).all() + if with_ids: + assert torch.count_nonzero(out[0]) == 0 + + +@pytest.mark.parametrize("strided_input", ["ids", "weight"]) +def test_post_expert_finalize_preserves_input_layout(strided_input): + """Small-batch finalization respects transposed IDs and sliced norm weights.""" + tokens, topk, hidden = 2, 2, 8 + x = torch.ones((tokens, topk, hidden), device="cuda", dtype=torch.bfloat16) + scores = torch.ones((tokens, topk), device="cuda") + ids = torch.tensor([[0, 0], [-1, -1]], device="cuda", dtype=torch.int32) + weight = torch.ones(hidden, device="cuda", dtype=torch.bfloat16) + if strided_input == "ids": + ids = ids.T.contiguous().T + assert not ids.is_contiguous() + else: + weight = torch.arange(1, 2 * hidden + 1, device="cuda", dtype=torch.bfloat16)[ + ::2 + ] + assert not weight.is_contiguous() + out = fused_post_expert_norm_weighted_sum( + x, scores, weight, eps=1e-6, slot_expert_ids=ids + ) + ref = _post_expert_reference(x, scores, weight, 1e-6, True, ids) + torch.testing.assert_close(out.float(), ref, rtol=2e-2, atol=2e-2) + assert torch.count_nonzero(out[1]) == 0 + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +def test_post_expert_fallback_preserves_product_rounding(dtype): + """Fallback rounds each product before summing and ignores poisoned slots.""" + from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import ( + finalize_identity_with_post_expert_norm, + ) + + one = torch.ones((), device="cuda", dtype=dtype) + a = torch.nextafter(one, torch.full_like(one, float("inf"))) + x = torch.empty((2, 3, 1), device="cuda", dtype=dtype) + x[0, :, 0] = torch.stack((a, -a, torch.full_like(a, float("nan")))) + x[1, :, 0] = torch.tensor( + [float("nan"), float("inf"), -float("inf")], device="cuda", dtype=dtype + ) + scores = torch.empty((2, 3), device="cuda", dtype=dtype) + scores[0] = torch.stack((a, one, torch.full_like(a, float("nan")))) + scores[1] = x[1, :, 0] + ids = torch.tensor([[0, 1, -1], [-1, -1, -1]], device="cuda", dtype=torch.int32) + + out = finalize_identity_with_post_expert_norm( + expert_outputs=x, + topk_scores=scores, + post_expert_norm=torch.nn.Identity(), + out_dtype=dtype, + slot_expert_ids=ids, + ) + expected = torch.stack((a * a, -a)).sum() + unrounded = (a.float() * a.float() - a.float()).to(dtype) + assert expected != unrounded + torch.testing.assert_close(out[0, 0], expected, rtol=0, atol=0) + assert torch.isfinite(out).all() + assert torch.count_nonzero(out[1]) == 0 if __name__ == "__main__":