From ac3db731b3f93e5af1896667ac0bf5734043af94 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:32:05 -0700 Subject: [PATCH] [Fix] moe: Preserve CUTLASS routing metadata and execution budgets (#45917) --- .../csrc/moe/cutlass_prepare_moe_input.cuh | 194 ++++ .../jit/csrc/moe/shuffle_weighted_sum.cuh | 142 +++ python/sglang/kernels/ops/moe/__init__.py | 20 +- .../ops/moe/cutlass_prepare_moe_input.py | 72 ++ .../sglang/kernels/ops/moe/ep_moe_kernels.py | 105 ++- .../kernels/ops/moe/shuffle_weighted_sum.py | 107 ++- python/sglang/srt/layers/moe/cutlass_moe.py | 24 +- .../sglang/srt/layers/moe/cutlass_w4a8_moe.py | 90 +- .../srt/layers/moe/flashinfer_trtllm_moe.py | 60 ++ .../srt/layers/moe/fused_moe_triton/layer.py | 13 +- python/sglang/srt/layers/moe/kt_ep_wrapper.py | 9 + .../moe/moe_runner/flashinfer_trtllm.py | 217 +++-- .../srt/layers/quantization/base_config.py | 8 +- python/sglang/srt/layers/quantization/fp8.py | 18 +- .../mxfp4_flashinfer_cutlass_moe.py | 6 + .../mxfp4_flashinfer_trtllm_moe.py | 5 + .../layers/quantization/mxfp4_humming_moe.py | 6 + .../layers/quantization/mxfp4_marlin_moe.py | 6 + .../sglang/srt/models/bailing_moe_linear.py | 5 +- python/sglang/srt/models/bailing_moe_v3.py | 5 +- python/sglang/srt/models/llada2.py | 5 +- .../moe/bench_shuffle_weighted_sum.py | 37 + .../moe/test_cutlass_fp8_inactive_routes.py | 870 +++++++++++++++++- .../moe/test_cutlass_w4a8_inactive_routes.py | 254 ++++- .../moe/test_flashinfer_fp8_routed.py | 294 ++++++ .../unit/kernels/test_kernels_namespace.py | 15 + .../layers/moe/test_standard_dispatcher.py | 63 +- ...test_standard_dispatcher_runner_backend.py | 120 ++- .../layers/moe/test_topk_routing_inputs.py | 370 +++++++- .../test_fp8_moe_runner_ownership.py | 172 +++- .../test_modelopt_nvfp4_moe_dispatch.py | 4 +- .../models/test_native_moe_routing_policy.py | 111 ++- 32 files changed, 3221 insertions(+), 206 deletions(-) create mode 100644 python/sglang/kernels/jit/csrc/moe/cutlass_prepare_moe_input.cuh create mode 100644 python/sglang/kernels/jit/csrc/moe/shuffle_weighted_sum.cuh create mode 100644 python/sglang/kernels/ops/moe/cutlass_prepare_moe_input.py create mode 100644 test/registered/kernels/benchmark/moe/bench_shuffle_weighted_sum.py create mode 100644 test/registered/moe/test_flashinfer_fp8_routed.py diff --git a/python/sglang/kernels/jit/csrc/moe/cutlass_prepare_moe_input.cuh b/python/sglang/kernels/jit/csrc/moe/cutlass_prepare_moe_input.cuh new file mode 100644 index 0000000000..6ed625a53e --- /dev/null +++ b/python/sglang/kernels/jit/csrc/moe/cutlass_prepare_moe_input.cuh @@ -0,0 +1,194 @@ +#pragma once + +#include + +#include + +#include +#include + +#include +#include +#include + +namespace sglang { +namespace cutlass_moe { + +template +__global__ void count_expert_routes( + const int32_t* __restrict__ topk_ids, + int32_t* __restrict__ problem_sizes1, + int32_t* __restrict__ problem_sizes2, + int32_t num_routes, + int32_t n, + int32_t k) { + __shared__ int32_t count; + if (threadIdx.x == 0) count = 0; + __syncthreads(); + + const int32_t expert = blockIdx.x; + int32_t local_count = 0; + for (uint32_t i = threadIdx.x; i < static_cast(num_routes); i += blockDim.x) { + local_count += topk_ids[i] == expert; + } + atomicAdd(&count, local_count); + __syncthreads(); + + if (threadIdx.x == 0) { + constexpr int kCountColumn = kTransposeProblemSizes ? 1 : 0; + constexpr int kOutputColumn = 1 - kCountColumn; + problem_sizes1[expert * 3 + kCountColumn] = count; + problem_sizes1[expert * 3 + kOutputColumn] = 2 * n; + problem_sizes1[expert * 3 + 2] = k; + problem_sizes2[expert * 3 + kCountColumn] = count; + problem_sizes2[expert * 3 + kOutputColumn] = k; + problem_sizes2[expert * 3 + 2] = n; + } +} + +template +__global__ void compute_expert_offsets( + const int32_t* __restrict__ problem_sizes1, + int32_t* __restrict__ expert_offsets, + int32_t* __restrict__ blockscale_offsets, + int32_t num_experts) { + int32_t offset = 0; + int32_t blockscale_offset = 0; + expert_offsets[0] = 0; + if (blockscale_offsets != nullptr) blockscale_offsets[0] = 0; + for (int32_t expert = 0; expert < num_experts; ++expert) { + constexpr int kCountColumn = kTransposeProblemSizes ? 1 : 0; + const int32_t count = problem_sizes1[expert * 3 + kCountColumn]; + offset += count; + expert_offsets[expert + 1] = offset; + if (blockscale_offsets != nullptr) { + blockscale_offset += (count + 127) / 128 * 128; + blockscale_offsets[expert + 1] = blockscale_offset; + } + } +} + +__global__ void complete_route_maps( + const int32_t* __restrict__ topk_ids, + const int32_t* __restrict__ expert_offsets, + int32_t* __restrict__ input_permutation, + int32_t* __restrict__ output_permutation, + int32_t num_routes, + int32_t topk, + int32_t num_experts) { + const int32_t expert = blockIdx.x; + // Exactly one CTA owns each expert's interval, so its cursor is block-local. + __shared__ int32_t cursor; + if (threadIdx.x == 0) cursor = expert_offsets[expert]; + __syncthreads(); + for (uint32_t i = threadIdx.x; i < static_cast(num_routes); i += blockDim.x) { + const int32_t route = topk_ids[i]; + if (route == expert) { + const int32_t destination = atomicAdd(&cursor, 1); + input_permutation[destination] = i / topk; + output_permutation[i] = destination; + } else if (expert == 0 && (route < 0 || route >= num_experts)) { + output_permutation[i] = -1; + } + } + + // The grouped GEMMs use only [0, live); full-capacity gathers also read the tail. + const int32_t live = expert_offsets[num_experts]; + const int64_t thread = static_cast(expert) * blockDim.x + threadIdx.x; + const int64_t stride = static_cast(num_experts) * blockDim.x; + for (int64_t i = live + thread; i < num_routes; i += stride) { + input_permutation[i] = 0; + } +} + +} // namespace cutlass_moe + +/// \brief Prepare CUTLASS grouped-GEMM shapes and complete route maps. +/// \param input_permutation Destination-to-token map, with zero-filled unused capacity. +/// \param output_permutation Route-to-destination map, with -1 for ids outside [0, num_experts). +/// \param blockscale_offsets Optional MXFP8 row offsets, rounded up to 128 per expert. +template +inline void cutlass_prepare_moe_input( + tvm::ffi::TensorView topk_ids, + tvm::ffi::TensorView expert_offsets, + tvm::ffi::TensorView problem_sizes1, + tvm::ffi::TensorView problem_sizes2, + tvm::ffi::TensorView input_permutation, + tvm::ffi::TensorView output_permutation, + int64_t num_experts, + int64_t n, + int64_t k, + tvm::ffi::Optional blockscale_offsets) { + using namespace host; + constexpr int64_t max_index = std::numeric_limits::max(); + CHECK_HOST(num_experts > 0 && num_experts <= max_index / 3); + CHECK_HOST(n > 0 && n <= max_index / 2 && k > 0 && k <= max_index); + + auto device = SymbolicDevice{}; + auto tokens = SymbolicSize{"tokens"}; + auto topk = SymbolicSize{"topk"}; + TensorMatcher({tokens, topk}) + .with_strides({-1, -1}) + .with_dtype() + .with_device(device) + .verify(topk_ids); + const int64_t num_routes = topk_ids.numel(); + // Empty PyTorch dimensions can have noncanonical strides; no ID is read. + RuntimeCheck(num_routes == 0 || topk_ids.is_contiguous(), "topk_ids must be contiguous when nonempty"); + CHECK_HOST(num_routes <= max_index); + TensorMatcher({num_experts + 1}).with_dtype().with_device(device).verify(expert_offsets); + TensorMatcher({num_experts, 3}) + .with_dtype() + .with_device(device) + .verify(problem_sizes1) + .verify(problem_sizes2); + TensorMatcher({num_routes}) + .with_dtype() + .with_device(device) + .verify(input_permutation) + .verify(output_permutation); + + int32_t* blockscale_ptr = nullptr; + if (blockscale_offsets.has_value()) { + TensorMatcher({num_experts + 1}).with_dtype().with_device(device).verify(blockscale_offsets.value()); + CHECK_HOST(num_routes + 127 * std::min(num_experts, num_routes) <= max_index); + blockscale_ptr = static_cast(blockscale_offsets.value().data_ptr()); + } + + const auto threads = static_cast(std::min(512, std::max(1, num_routes))); + const auto experts = static_cast(num_experts); + const auto routes = static_cast(num_routes); + const auto* ids = static_cast(topk_ids.data_ptr()); + auto* sizes1 = static_cast(problem_sizes1.data_ptr()); + auto* sizes2 = static_cast(problem_sizes2.data_ptr()); + auto* offsets = static_cast(expert_offsets.data_ptr()); + const auto stream = LaunchKernel::resolve_device(device.unwrap()); + + LaunchKernel(dim3(experts), dim3(threads), stream)( + cutlass_moe::count_expert_routes, + ids, + sizes1, + sizes2, + routes, + static_cast(n), + static_cast(k)); + LaunchKernel(dim3(1), dim3(1), stream)( + cutlass_moe::compute_expert_offsets, + sizes1, + offsets, + blockscale_ptr, + static_cast(num_experts)); + if (num_routes != 0) { + LaunchKernel(dim3(experts), dim3(threads), stream)( + cutlass_moe::complete_route_maps, + ids, + offsets, + static_cast(input_permutation.data_ptr()), + static_cast(output_permutation.data_ptr()), + routes, + static_cast(topk.unwrap()), + static_cast(num_experts)); + } +} + +} // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/moe/shuffle_weighted_sum.cuh b/python/sglang/kernels/jit/csrc/moe/shuffle_weighted_sum.cuh new file mode 100644 index 0000000000..9a64e2dd68 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/moe/shuffle_weighted_sum.cuh @@ -0,0 +1,142 @@ +#pragma once + +#include + +#include +#include + +#include +#include +#include + +namespace sglang { +namespace shuffle_weighted_sum { + +SGL_DEVICE float accumulate(float value, float factor, float total) { + float result; + // Keep the slot-order FMA, including +0 * +0 for inactive slots. No FTZ. + asm volatile("fma.rn.f32 %0, %1, %2, %3;" : "=f"(result) : "f"(value), "f"(factor), "f"(total)); + return result; +} + +template < + typename scalar_t, + typename weight_t, + typename index_t, + int64_t kTopK, + int kThreads, + bool kCastWeightsToInputDType> +__global__ void reduce( + const scalar_t* inputs, + scalar_t* output, + const index_t* permutation, + const weight_t* weights, + int64_t width, + int64_t rows, + int64_t input_stride, + int64_t output_stride, + int64_t weight_stride0, + int64_t weight_stride1) { + constexpr int kItems = 2; + const int64_t token = blockIdx.x; + const int64_t first_column = static_cast(blockIdx.y) * kThreads * kItems + threadIdx.x; + if (first_column >= width) return; + float totals[kItems] = {0.0f, 0.0f}; + for (int64_t slot = 0; slot < kTopK; ++slot) { + const int64_t row = permutation[token * kTopK + slot]; + const bool active = row >= 0 && row < rows; + float factor = 0.0f; + if (active) { + factor = device::cast(weights[token * weight_stride0 + slot * weight_stride1]); + if constexpr (kCastWeightsToInputDType && !std::is_same_v) { + // Match scores.to(inputs.dtype) before the FP32 slot-order FMA. + factor = device::cast(device::cast(factor)); + } + } +#pragma unroll + for (int item = 0; item < kItems; ++item) { + const int64_t column = first_column + item * kThreads; + float value = 0.0f; + if (active && column < width) value = device::cast(inputs[row * input_stride + column]); + totals[item] = accumulate(value, factor, totals[item]); + } + } +#pragma unroll + for (int item = 0; item < kItems; ++item) { + const int64_t column = first_column + item * kThreads; + if (column < width) output[token * output_stride + column] = device::cast(totals[item]); + } +} + +} // namespace shuffle_weighted_sum + +/// \brief Reduce mapped rows in slot order without reading inactive payloads/scores. +/// \param permutation Contiguous int32/int64 map of any rank, one entry per weight. +/// \param weights Potentially strided scores; rows outside the input extent are inactive. +template < + typename scalar_t, + typename weight_t, + typename index_t, + int64_t kTopK, + int kThreads, + bool kCastWeightsToInputDType> +void shuffle_weighted_sum_cuda( + tvm::ffi::TensorView inputs, + tvm::ffi::TensorView output, + tvm::ffi::TensorView permutation, + tvm::ffi::TensorView weights) { + using namespace host; + static_assert(std::is_same_v || std::is_same_v); + static_assert( + std::is_same_v || std::is_same_v || std::is_same_v); + static_assert(std::is_same_v || std::is_same_v); + static_assert(kTopK >= 0 && kThreads > 0 && kThreads <= 1024); + auto device = SymbolicDevice{}; + device.set_options(); + auto rows = SymbolicSize{"input_rows"}; + auto tokens = SymbolicSize{"output_rows"}; + auto width = SymbolicSize{"width"}; + auto input_stride = SymbolicSize{"input_stride"}; + auto output_stride = SymbolicSize{"output_stride"}; + auto weight_stride0 = SymbolicSize{"weight_stride0"}; + auto weight_stride1 = SymbolicSize{"weight_stride1"}; + TensorMatcher({rows, width}) + .with_strides({input_stride, 1}) + .with_dtype() + .with_device(device) + .verify(inputs); + TensorMatcher({tokens, width}) + .with_strides({output_stride, 1}) + .with_dtype() + .with_device(device) + .verify(output); + TensorMatcher({tokens, kTopK}) + .with_strides({weight_stride0, weight_stride1}) + .with_dtype() + .with_device(device) + .verify(weights); + auto map_dtype = SymbolicDType{}; + map_dtype.set_options(); + map_dtype.verify(permutation.dtype()); + device.verify(permutation.device()); + CHECK_HOST(permutation.is_contiguous() && permutation.numel() == weights.numel()); + if (tokens.unwrap() == 0 || width.unwrap() == 0) return; + constexpr int64_t kTile = kThreads * 2; + const int64_t tiles = div_ceil(width.unwrap(), kTile); + // These are the same x/y grid limits as the Triton fallback. + CHECK_HOST(tokens.unwrap() <= std::numeric_limits::max() && tiles <= 65535); + LaunchKernel(dim3(static_cast(tokens.unwrap()), static_cast(tiles)), kThreads, device.unwrap())( + shuffle_weighted_sum::reduce, + static_cast(inputs.data_ptr()), + static_cast(output.data_ptr()), + static_cast(permutation.data_ptr()), + static_cast(weights.data_ptr()), + width.unwrap(), + rows.unwrap(), + input_stride.unwrap(), + output_stride.unwrap(), + weight_stride0.unwrap(), + weight_stride1.unwrap()); +} + +} // namespace sglang diff --git a/python/sglang/kernels/ops/moe/__init__.py b/python/sglang/kernels/ops/moe/__init__.py index b3a4f6672c..d519c41efb 100644 --- a/python/sglang/kernels/ops/moe/__init__.py +++ b/python/sglang/kernels/ops/moe/__init__.py @@ -71,10 +71,19 @@ register_kernel( ) +register_kernel( + KernelSpec( + op="moe.cutlass_prepare_moe_input", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.moe.cutlass_prepare_moe_input:cutlass_prepare_moe_input", + capabilities=CUDA_ONLY, + ) +) + register_kernel( KernelSpec( op="moe.shuffle_weighted_sum", - backend=KernelBackend.TRITON, + backend=KernelBackend.JIT, target="sglang.kernels.ops.moe.shuffle_weighted_sum:shuffle_weighted_sum", capabilities=CUDA_ONLY, ) @@ -112,6 +121,15 @@ register_kernel( ) ) +register_kernel( + KernelSpec( + op="moe.compute_cutlass_w4a8_moe_input_scale", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.moe.ep_moe_kernels:compute_cutlass_w4a8_moe_input_scale", + capabilities=CUDA_ONLY, + ) +) + def moe_align_block_size( topk_ids: torch.Tensor, diff --git a/python/sglang/kernels/ops/moe/cutlass_prepare_moe_input.py b/python/sglang/kernels/ops/moe/cutlass_prepare_moe_input.py new file mode 100644 index 0000000000..0334ec3629 --- /dev/null +++ b/python/sglang/kernels/ops/moe/cutlass_prepare_moe_input.py @@ -0,0 +1,72 @@ +"""Complete routing maps and grouped-GEMM shapes for CUTLASS MoE.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +import torch + +from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args +from sglang.srt.utils.custom_op import register_custom_op + +if TYPE_CHECKING: + from tvm_ffi.module import Module + + +@cache_once +def _jit_cutlass_prepare_moe_input_module( + transpose_problem_sizes: bool = False, +) -> Module: + args = make_cpp_args(transpose_problem_sizes) + return load_jit( + "cutlass_prepare_moe_input", + *args, + cuda_files=["moe/cutlass_prepare_moe_input.cuh"], + cuda_wrappers=[("prepare", f"cutlass_prepare_moe_input<{args}>")], + ) + + +@register_custom_op( + op_name="cutlass_prepare_moe_input", + mutates_args=[ + "expert_offsets", + "problem_sizes1", + "problem_sizes2", + "input_permutation", + "output_permutation", + "blockscale_offsets", + ], +) +def cutlass_prepare_moe_input( + topk_ids: torch.Tensor, + expert_offsets: torch.Tensor, + problem_sizes1: torch.Tensor, + problem_sizes2: torch.Tensor, + input_permutation: torch.Tensor, + output_permutation: torch.Tensor, + num_experts: int, + n: int, + k: int, + blockscale_offsets: Optional[torch.Tensor] = None, + *, + transpose_problem_sizes: bool = False, +) -> None: + """Write contiguous int32 CUTLASS metadata, including every unused map entry. + + Routes outside [0, num_experts) have output permutation -1. Input-map + capacity beyond expert_offsets[-1] points at token zero for full gathers. + Optional blockscale offsets count per-expert rows rounded up to 128. + Transposed problem sizes put the token count in N instead of M. + """ + _jit_cutlass_prepare_moe_input_module(transpose_problem_sizes).prepare( + topk_ids, + expert_offsets, + problem_sizes1, + problem_sizes2, + input_permutation, + output_permutation, + num_experts, + n, + k, + blockscale_offsets, + ) diff --git a/python/sglang/kernels/ops/moe/ep_moe_kernels.py b/python/sglang/kernels/ops/moe/ep_moe_kernels.py index 0b6d96469b..7397221602 100644 --- a/python/sglang/kernels/ops/moe/ep_moe_kernels.py +++ b/python/sglang/kernels/ops/moe/ep_moe_kernels.py @@ -145,13 +145,27 @@ def deepep_post_reorder_triton_kernel( @triton.jit def compute_src2dst_triton_kernel( - reorder_ids, src2dst, num_toks, BLOCK_SIZE: tl.constexpr + sorted_topk_ids, reorder_ids, src2dst, num_toks, BLOCK_SIZE: tl.constexpr ): + first_nonnegative = 0 + if tl.load(sorted_topk_ids) < 0: + left, right = 0, num_toks + while left < right: + middle = left + (right - left) // 2 + expert_id = tl.load(sorted_topk_ids + middle) + if expert_id < 0: + left = middle + 1 + else: + right = middle + first_nonnegative = left + pid = tl.program_id(axis=0) dst_id = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) mask = dst_id < num_toks src_id = tl.load(reorder_ids + dst_id, mask=mask) - tl.store(src2dst + src_id, dst_id, mask=mask) + # Native expert offsets omit negative IDs, so packed rows must omit that prefix. + packed_id = tl.where(dst_id >= first_nonnegative, dst_id - first_nonnegative, -1) + tl.store(src2dst + src_id, packed_id, mask=mask) @triton.jit @@ -189,18 +203,95 @@ def deepep_run_moe_deep_preprocess(topk_ids: torch.Tensor, num_experts: int): def cutlass_w4_run_moe_ep_preproess(topk_ids: torch.Tensor): - _, reorder_ids = torch.sort(topk_ids.view(-1), stable=True) + src2dst = torch.empty(topk_ids.numel(), device=topk_ids.device, dtype=torch.int32) + if topk_ids.numel() == 0: + return src2dst + sorted_topk_ids, reorder_ids = torch.sort(topk_ids.view(-1), stable=True) BLOCK_SIZE = 512 grid = (triton.cdiv(topk_ids.numel(), BLOCK_SIZE),) - src2dst = torch.empty(topk_ids.numel(), device=topk_ids.device, dtype=torch.int32) compute_src2dst_triton_kernel[grid]( - reorder_ids, src2dst, topk_ids.numel(), BLOCK_SIZE + sorted_topk_ids, reorder_ids, src2dst, topk_ids.numel(), BLOCK_SIZE ) return src2dst +@triton.jit +def _compute_cutlass_w4a8_moe_input_scale_kernel( + input_ptr, + topk_ids_ptr, + output_s_ptr, + stride_input_row, + stride_input_col, + stride_ids_row, + stride_ids_col, + hidden_size: tl.constexpr, + topk: tl.constexpr, + num_local_experts: tl.constexpr, + FP8_MAX: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + BLOCK_TOPK: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + routes = tl.arange(0, BLOCK_TOPK) + expert_ids = tl.load( + topk_ids_ptr + row * stride_ids_row + routes * stride_ids_col, + mask=routes < topk, + other=-1, + ) + local_route = (expert_ids >= 0) & (expert_ids < num_local_experts) + active = tl.sum(local_route.to(tl.int32), 0) > 0 + columns = tl.arange(0, BLOCK_SIZE) + values = tl.load( + input_ptr + row * stride_input_row + columns * stride_input_col, + mask=(columns < hidden_size) & active, + other=0.0, + ).to(tl.float32) + absmax = tl.max(tl.abs(values), 0) + tl.atomic_max(output_s_ptr, absmax / FP8_MAX, sem="relaxed") + + +def compute_cutlass_w4a8_moe_input_scale( + input: torch.Tensor, + topk_ids: torch.Tensor, + output_s: torch.Tensor, + num_local_experts: int, +): + """Accumulate the FP8 scale from rows with a local expert route. + + The caller initializes output_s to its minimum finite scale. Inputs and IDs + are preserved, and fully inactive rows never load activations, including NaNs. + """ + assert input.ndim == 2 + assert input.dtype in (torch.float16, torch.bfloat16, torch.float32) + assert topk_ids.ndim == 2 and topk_ids.shape[0] == input.shape[0] + assert topk_ids.dtype in (torch.int32, torch.int64) + assert output_s.numel() == 1 and output_s.dtype == torch.float32 + assert input.is_cuda + assert input.device == topk_ids.device == output_s.device + if input.numel() == 0 or topk_ids.shape[1] == 0: + return + hidden_size = input.shape[1] + block_size = triton.next_power_of_2(max(1, hidden_size)) + _compute_cutlass_w4a8_moe_input_scale_kernel[(input.shape[0],)]( + input, + topk_ids, + output_s, + input.stride(0), + input.stride(1), + topk_ids.stride(0), + topk_ids.stride(1), + hidden_size=hidden_size, + topk=topk_ids.shape[1], + num_local_experts=num_local_experts, + FP8_MAX=torch.finfo(torch.float8_e4m3fn).max, + BLOCK_SIZE=block_size, + BLOCK_TOPK=triton.next_power_of_2(topk_ids.shape[1]), + num_warps=4 if block_size < 4096 else 8, + ) + + @triton.jit def pre_reorder_triton_kernel_for_cutlass_moe( input_ptr, @@ -241,7 +332,7 @@ def pre_reorder_triton_kernel_for_cutlass_moe( out_data = (in_data * a1_scale).to(OutDtype) for idx in range(topk): expert_id = tl.load(token_topk_ids_ptr + idx) - if expert_id != num_local_experts: + if (expert_id >= 0) & (expert_id < num_local_experts): dst_idx = tl.load(token_src2dst_ptr + idx).to(tl.int64) tl.store(dst_ptr_offs + dst_idx * hidden_size, out_data, mask=mask) @@ -855,7 +946,7 @@ def post_reorder_triton_kernel_for_cutlass_moe( sum_vec = tl.zeros([BLOCK_SIZE], dtype=tl.float32) for idx in range(topk): expert_id = tl.load(token_topk_ids_ptr + idx) - if expert_id != num_local_experts: + if (expert_id >= 0) & (expert_id < num_local_experts): dst_idx_int32 = tl.load(token_src2dst_ptr + idx) dst_idx = dst_idx_int32.to(tl.int64) dst_idx = dst_idx diff --git a/python/sglang/kernels/ops/moe/shuffle_weighted_sum.py b/python/sglang/kernels/ops/moe/shuffle_weighted_sum.py index 4a1b51235a..17819f0622 100644 --- a/python/sglang/kernels/ops/moe/shuffle_weighted_sum.py +++ b/python/sglang/kernels/ops/moe/shuffle_weighted_sum.py @@ -1,9 +1,56 @@ """Reduce routed rows while skipping inactive permutation entries.""" +from __future__ import annotations + +from typing import TYPE_CHECKING + import torch import triton import triton.language as tl +from sglang.kernels.jit.utils import ( + cache_once, + is_hip_runtime, + is_musa_runtime, + load_jit, + make_cpp_args, +) +from sglang.kernels.jit.utils.common import is_sail_runtime +from sglang.srt.utils.custom_op import register_custom_op + +if TYPE_CHECKING: + from tvm_ffi.module import Module + + +@cache_once +def _jit_shuffle_weighted_sum_module( + dtype: torch.dtype, + output_dtype: torch.dtype, + weight_dtype: torch.dtype, + index_dtype: torch.dtype, + topk: int, + cast_weights_to_input_dtype: bool = False, +) -> Module | None: + if ( + dtype not in (torch.float16, torch.bfloat16) + or output_dtype != dtype + or weight_dtype not in (torch.float16, torch.bfloat16, torch.float32) + or index_dtype not in (torch.int32, torch.int64) + or is_hip_runtime() + or is_musa_runtime() + or is_sail_runtime() + ): + return None + args = make_cpp_args( + dtype, weight_dtype, index_dtype, topk, 128, cast_weights_to_input_dtype + ) + return load_jit( + "shuffle_weighted_sum", + *args, + cuda_files=["moe/shuffle_weighted_sum.cuh"], + cuda_wrappers=[("reduce", f"shuffle_weighted_sum_cuda<{args}>")], + ) + @triton.jit(do_not_specialize=["rows", "weight_stride0", "weight_stride1"]) def _shuffle_weighted_sum_kernel( @@ -19,6 +66,7 @@ def _shuffle_weighted_sum_kernel( weight_stride0, weight_stride1, BLOCK: tl.constexpr, + CAST_WEIGHTS_TO_INPUT_DTYPE: tl.constexpr = False, ): token = tl.program_id(0) columns = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) @@ -35,7 +83,10 @@ def _shuffle_weighted_sum_kernel( weights + token * weight_stride0 + slot * weight_stride1, mask=active, other=0, - ).to(tl.float32) + ) + if CAST_WEIGHTS_TO_INPUT_DTYPE: + factor = factor.to(inputs.dtype.element_ty) + factor = factor.to(tl.float32) total += values * factor tl.store( output + token.to(tl.int64) * output_stride + columns, @@ -44,13 +95,15 @@ def _shuffle_weighted_sum_kernel( ) -def shuffle_weighted_sum( +def _shuffle_weighted_sum_triton( inputs: torch.Tensor, output: torch.Tensor, permutation: torch.Tensor, weights: torch.Tensor, + *, + cast_weights_to_input_dtype: bool = False, ) -> None: - """Sum mapped expert outputs; inactive maps read neither data nor weights.""" + """Sum mapped rows, optionally rounding scores to the input dtype first.""" assert inputs.ndim == output.ndim == weights.ndim == 2 assert inputs.shape[1] == output.shape[1] assert inputs.stride(1) == output.stride(1) == 1 @@ -58,6 +111,17 @@ def shuffle_weighted_sum( assert permutation.is_contiguous() and permutation.numel() == weights.numel() if output.numel() == 0: return + if cast_weights_to_input_dtype and ( + inputs.dtype not in (torch.float16, torch.bfloat16) + or weights.dtype not in (torch.float16, torch.bfloat16, torch.float32) + or not inputs.is_cuda + or is_hip_runtime() + or is_musa_runtime() + or is_sail_runtime() + ): + # Preserve Torch conversion semantics outside the fused CUDA types. + weights = weights.to(inputs.dtype) + cast_weights_to_input_dtype = False _shuffle_weighted_sum_kernel[(output.shape[0], triton.cdiv(output.shape[1], 256))]( inputs, output, @@ -71,4 +135,41 @@ def shuffle_weighted_sum( weights.stride(0), weights.stride(1), BLOCK=256, + CAST_WEIGHTS_TO_INPUT_DTYPE=cast_weights_to_input_dtype, + ) + + +@register_custom_op(op_name="shuffle_weighted_sum", mutates_args=["output"]) +def shuffle_weighted_sum( + inputs: torch.Tensor, + output: torch.Tensor, + permutation: torch.Tensor, + weights: torch.Tensor, + *, + cast_weights_to_input_dtype: bool = False, +) -> None: + """Sum mapped rows, optionally rounding scores to the input dtype first.""" + if ( + inputs.is_cuda + and inputs.dtype in (torch.float16, torch.bfloat16) + and inputs.ndim == output.ndim == weights.ndim == 2 + and output.numel() != 0 + ): + module = _jit_shuffle_weighted_sum_module( + inputs.dtype, + output.dtype, + weights.dtype, + permutation.dtype, + weights.shape[1], + cast_weights_to_input_dtype, + ) + if module is not None: + module.reduce(inputs, output, permutation, weights) + return + _shuffle_weighted_sum_triton( + inputs, + output, + permutation, + weights, + cast_weights_to_input_dtype=cast_weights_to_input_dtype, ) diff --git a/python/sglang/srt/layers/moe/cutlass_moe.py b/python/sglang/srt/layers/moe/cutlass_moe.py index b2b8ffa3cd..bfbeb5a7db 100755 --- a/python/sglang/srt/layers/moe/cutlass_moe.py +++ b/python/sglang/srt/layers/moe/cutlass_moe.py @@ -14,11 +14,13 @@ if _is_cuda: es_sm100_mxfp8_blockscaled_grouped_mm, es_sm100_mxfp8_blockscaled_grouped_quant, fp8_blockwise_scaled_grouped_mm, - prepare_moe_input, shuffle_rows, ) from sglang.kernels.ops.activation.activation import silu_and_mul + from sglang.kernels.ops.moe.cutlass_prepare_moe_input import ( + cutlass_prepare_moe_input, + ) from sglang.kernels.ops.moe.shuffle_rows_with_scales import ( shuffle_rows_with_scales, ) @@ -142,10 +144,6 @@ def cutlass_fused_experts_fp8( topk = topk_ids.size(1) device = a.device - # prepare_moe_input writes only live routes; gathers still span the tail. - a_map = torch.zeros((topk_ids.numel()), dtype=torch.int32, device=device) - c_map = torch.full((topk_ids.numel(),), -1, dtype=torch.int32, device=device) - if use_mxfp8: assert es_up and es_down, "MXFP8 requires expert-specialization for both GEMMs" assert get_platform().is_sm100, "MXFP8 requires SM100" @@ -178,13 +176,23 @@ def cutlass_fused_experts_fp8( (max_total + mxfp8_blockscale_align - 1) // mxfp8_blockscale_align ) * mxfp8_blockscale_align + if m == 0 or topk == 0: + if output is None: + output = torch.empty((m, k), dtype=out_dtype, device=device) + if m != 0: + output.zero_() + return output + + a_map = torch.empty((topk_ids.numel(),), dtype=torch.int32, device=device) + c_map = torch.empty((topk_ids.numel(),), dtype=torch.int32, device=device) + blockscale_offsets = None if use_mxfp8 and (es_up or es_down): blockscale_offsets = torch.empty( (num_experts + 1,), dtype=torch.int32, device=device ) - prepare_moe_input( + cutlass_prepare_moe_input( topk_ids, expert_offsets, problem_sizes1, @@ -346,5 +354,7 @@ def cutlass_fused_experts_fp8( if output is None: output = torch.empty((m, k), device=device, dtype=out_dtype) - shuffle_weighted_sum(c2, output, c_map, topk_weights.to(out_dtype)) + shuffle_weighted_sum( + c2, output, c_map, topk_weights, cast_weights_to_input_dtype=True + ) return output diff --git a/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py b/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py index f6b3c08326..e5dbcb0637 100644 --- a/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py +++ b/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py @@ -18,10 +18,14 @@ if _is_cuda_alike: if _is_cuda: from sglang.kernels.ops.activation.activation import silu_and_mul + from sglang.kernels.ops.moe.cutlass_prepare_moe_input import ( + cutlass_prepare_moe_input, + ) else: from sgl_kernel import silu_and_mul from sglang.kernels.ops.moe.ep_moe_kernels import ( + compute_cutlass_w4a8_moe_input_scale, cutlass_w4_run_moe_ep_preproess, deepep_ll_get_cutlass_w4a8_moe_mm_data, deepep_permute_triton_kernel, @@ -34,10 +38,7 @@ from sglang.kernels.ops.moe.ep_moe_kernels import ( silu_mul_dynamic_tensorwise_quant_for_cutlass_moe, silu_mul_static_tensorwise_quant_for_cutlass_moe, ) -from sglang.kernels.ops.quantization.per_tensor_quant_fp8 import ( - per_tensor_absmax_fp8, - per_tensor_quant_fp8, -) +from sglang.kernels.ops.quantization.per_tensor_quant_fp8 import per_tensor_quant_fp8 def cutlass_w4a8_moe( @@ -85,6 +86,7 @@ def cutlass_w4a8_moe( Shape: [num_experts, N // 512, K * 4] - topk_weights (torch.Tensor): The weights of each token->expert mapping. - topk_ids (torch.Tensor): The ids of each token->expert mapping. + Noncontiguous IDs are materialized once for the native flat-pointer consumers. - a_strides1 (torch.Tensor): The input strides of the first grouped gemm. - b_strides1 (torch.Tensor): The weights strides of the first grouped gemm. - c_strides1 (torch.Tensor): The output strides of the first grouped gemm. @@ -126,13 +128,42 @@ def cutlass_w4a8_moe( if apply_router_weight_on_input: assert topk == 1, "apply_router_weight_on_input is only implemented for topk=1" - device = a.device - # Both graph padding and nonlocal routes use -1, including without EP. - topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids) + # Native expert counts and reorder kernels read contiguous int32 IDs. + topk_ids = topk_ids.contiguous() - src2dst = cutlass_w4_run_moe_ep_preproess( - topk_ids, - ) + device = a.device + if a1_scale is None: + a1_scale = torch.full( + (1,), + 1e-10 / torch.finfo(torch.float8_e4m3fn).max, + dtype=torch.float32, + device=device, + ) + compute_cutlass_w4a8_moe_input_scale( + input=a, + topk_ids=topk_ids, + output_s=a1_scale, + num_local_experts=num_local_experts, + ) + + if _is_cuda: + a_map = torch.empty(topk_ids.numel(), dtype=torch.int32, device=device) + c_map = torch.empty(topk_ids.numel(), dtype=torch.int32, device=device) + cutlass_prepare_moe_input( + topk_ids, + expert_offsets, + problem_sizes1, + problem_sizes2, + a_map, + c_map, + num_local_experts, + n, + k, + transpose_problem_sizes=True, + ) + src2dst = c_map + else: + src2dst = cutlass_w4_run_moe_ep_preproess(topk_ids) gateup_input = torch.empty( (m * topk, k), @@ -140,15 +171,6 @@ def cutlass_w4a8_moe( dtype=torch.float8_e4m3fn, ) - # TODO: fuse per_tensor_absmax_fp8 and pre_reorder_for_cutlass_moe - if a1_scale is None: - a1_scale = torch.zeros(1, dtype=torch.float32, device=device) - # Inactive rows must not change the quantization of live expert inputs. - inactive_rows = (topk_ids == num_local_experts).all(dim=-1, keepdim=True) - scale_input = a.masked_fill(inactive_rows, 0.0) - per_tensor_absmax_fp8(scale_input, a1_scale) - a1_scale.clamp_(min=1e-10 / torch.finfo(torch.float8_e4m3fn).max) - pre_reorder_for_cutlass_moe( a, gateup_input, @@ -161,22 +183,20 @@ def cutlass_w4a8_moe( k, ) - # NOTE: a_map and c_map are not used in the get_cutlass_w4a8_moe_mm_data kernel, - # they are kept to allow for a quick switch of the permutation logic - # from the current triton kernel implementation to the cutlass-based one if needed. - a_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device) - c_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device) - get_cutlass_w4a8_moe_mm_data( - topk_ids, - expert_offsets, - problem_sizes1, - problem_sizes2, - a_map, - c_map, - num_local_experts, - n, - k, - ) + if not _is_cuda: + a_map = torch.empty(topk_ids.numel(), dtype=torch.int32, device=device) + c_map = torch.empty(topk_ids.numel(), dtype=torch.int32, device=device) + get_cutlass_w4a8_moe_mm_data( + topk_ids, + expert_offsets, + problem_sizes1, + problem_sizes2, + a_map, + c_map, + num_local_experts, + n, + k, + ) c1 = torch.empty((m * topk, n * 2), device=device, dtype=torch.bfloat16) c2 = torch.empty((m * topk, k), device=device, dtype=torch.bfloat16) diff --git a/python/sglang/srt/layers/moe/flashinfer_trtllm_moe.py b/python/sglang/srt/layers/moe/flashinfer_trtllm_moe.py index d7f50b1731..c180669b3a 100644 --- a/python/sglang/srt/layers/moe/flashinfer_trtllm_moe.py +++ b/python/sglang/srt/layers/moe/flashinfer_trtllm_moe.py @@ -323,3 +323,63 @@ def trtllm_fp8_per_tensor_scale_moe_wrapper( kwargs["activation_type"] = ActivationType(activation_type) return trtllm_fp8_per_tensor_scale_moe(**kwargs) + + +@register_custom_op(mutates_args=["output"]) +def trtllm_fp8_per_tensor_scale_routed_moe_out_wrapper( + topk_ids: torch.Tensor, + topk_weights: Optional[torch.Tensor], + hidden_states: torch.Tensor, + gemm1_weights: torch.Tensor, + output1_scales_scalar: torch.Tensor, + output1_scales_gate_scalar: torch.Tensor, + gemm2_weights: torch.Tensor, + output2_scales_scalar: torch.Tensor, + output: torch.Tensor, + num_experts: int, + top_k: int, + intermediate_size: int, + local_expert_offset: int, + local_num_experts: int, + routed_scaling_factor: Optional[float], + routing_method_type: int, + enable_pdl: Optional[bool] = None, + tune_max_num_tokens: int = 8192, + activation_type: Optional[int] = None, +) -> None: + try: + from flashinfer.fused_moe import trtllm_fp8_per_tensor_scale_routed_moe + except ImportError as e: + raise ImportError( + "FlashInfer TRT-LLM per-tensor FP8 with materialized routing requires " + "trtllm_fp8_per_tensor_scale_routed_moe; use the pinned CUDA FlashInfer build." + ) from e + + kwargs = dict( + topk_ids=topk_ids if topk_weights is None else (topk_ids, topk_weights), + routing_bias=None, + hidden_states=hidden_states, + gemm1_weights=gemm1_weights, + output1_scales_scalar=output1_scales_scalar, + output1_scales_gate_scalar=output1_scales_gate_scalar, + gemm2_weights=gemm2_weights, + output2_scales_scalar=output2_scales_scalar, + num_experts=num_experts, + top_k=top_k, + n_group=None, + topk_group=None, + intermediate_size=intermediate_size, + local_expert_offset=local_expert_offset, + local_num_experts=local_num_experts, + routed_scaling_factor=routed_scaling_factor, + use_routing_scales_on_input=False, + routing_method_type=routing_method_type, + output=output, + enable_pdl=enable_pdl, + tune_max_num_tokens=tune_max_num_tokens, + ) + if activation_type is not None: + from flashinfer.fused_moe.core import ActivationType + + kwargs["activation_type"] = ActivationType(activation_type) + trtllm_fp8_per_tensor_scale_routed_moe(**kwargs) diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index f15e6579f0..e06adbd486 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -604,7 +604,10 @@ class FusedMoE(torch.nn.Module): _fuses_routed_scaling_factor_in_topk(self.quant_method, self.runner_backend) ) - self.routing_method_type = routing_method_type + self.routing_method_type = self.quant_method.get_moe_routing_method_type( + self, routing_method_type + ) + self.moe_runner_config.routing_method_type = self.routing_method_type # overlap args self.down_gemm_overlap_args: Optional[DownGemmOverlapArgs] = None @@ -1564,11 +1567,11 @@ class FusedMoE(torch.nn.Module): return forward_fuseep(self, hidden_states, topk_output) if is_in_tc_piecewise_cuda_graph(): - if ( - TopKOutputChecker.format_is_bypassed(topk_output) - and topk_output.expert_location_dispatch_info is not None + if TopKOutputChecker.format_is_bypassed(topk_output) and ( + topk_output.expert_location_dispatch_info is not None + or topk_output.num_token_non_padded is not None ): - # The opaque logits-only op cannot carry placement metadata. + # The opaque logits-only op cannot carry padding or placement metadata. return self.forward_impl( hidden_states, topk_output, pre_quant_input=pre_quant_input ) diff --git a/python/sglang/srt/layers/moe/kt_ep_wrapper.py b/python/sglang/srt/layers/moe/kt_ep_wrapper.py index 1849fb0fa0..bb4d54d7be 100644 --- a/python/sglang/srt/layers/moe/kt_ep_wrapper.py +++ b/python/sglang/srt/layers/moe/kt_ep_wrapper.py @@ -275,6 +275,9 @@ class KTEPWrapperMethod(FusedMoEMethodBase): def get_moe_runner_backend(self, layer): return self.gpu_method.get_moe_runner_backend(layer) + def get_moe_routing_method_type(self, layer, requested_method): + return self.gpu_method.get_moe_routing_method_type(layer, requested_method) + def submit( self, layer: torch.nn.Module, @@ -344,6 +347,12 @@ class KTEPWrapperMethod(FusedMoEMethodBase): Combined computation results from CPU and GPU experts """ from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + from sglang.srt.layers.moe.topk import TopKOutputChecker + + if TopKOutputChecker.format_is_bypassed(dispatch_output.topk_output): + dispatch_output = dispatch_output._replace( + topk_output=dispatch_output.topk_output.to_standard() + ) x = dispatch_output.hidden_states topk_output = dispatch_output.topk_output diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py index 631dd2a06b..6bdaa00f04 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py @@ -26,6 +26,7 @@ from sglang.srt.layers.moe.flashinfer_trtllm_moe import ( trtllm_fp8_block_scale_moe_out_wrapper, trtllm_fp8_block_scale_routed_moe_out_wrapper, trtllm_fp8_per_tensor_scale_moe_wrapper, + trtllm_fp8_per_tensor_scale_routed_moe_out_wrapper, ) from sglang.srt.layers.moe.moe_runner.base import ( MoeQuantInfo, @@ -103,7 +104,6 @@ def _make_deferred_finalize_output( result, *, top_k: int, - materialized_routing: bool = False, ) -> FlashInferTrtllmDeferredFinalizeOutput: """Validate and adapt FlashInfer's ``do_finalize=False`` output ABI.""" gemm2_out, expert_weights, expanded_idx_to_permuted_idx = result[:3] @@ -178,18 +178,24 @@ def _is_gated(layer: Module) -> bool: FlashInferRouting = torch.Tensor | tuple[torch.Tensor, torch.Tensor] -def _get_routing_for_flashinfer_routed(topk_output) -> FlashInferRouting: - """Return the `topk_ids` kernel argument for the trtllm routed MoEs.""" +def _get_routing_for_flashinfer_routed( + topk_output, *, promote_half_scores: bool = False +) -> FlashInferRouting: + """Return routed IDs/scores, optionally widening FP16 for native FP8 kernels.""" from sglang.srt.layers.moe.topk import TopKOutputChecker if TopKOutputChecker.format_is_packed(topk_output): return topk_output.packed_topk_ids assert TopKOutputChecker.format_is_standard(topk_output) - return ( - topk_output.topk_ids.contiguous(), - topk_output.topk_weights.contiguous(), - ) + weights = topk_output.topk_weights + if promote_half_scores and weights.dtype == torch.float16: + # FP8 providers accept BF16/FP32 scores. Widen and make contiguous in + # one copy, retaining the values already rounded by the router. + weights = weights.to(dtype=torch.float32, memory_format=torch.contiguous_format) + else: + weights = weights.contiguous() + return topk_output.topk_ids.contiguous(), weights def _routing_top_k(routing: FlashInferRouting) -> int: @@ -771,6 +777,85 @@ class FlashInferTrtllmFp8MoeQuantInfo(MoeQuantInfo): activation_type: int | None = None +def _run_fp8_per_tensor_scale_moe( + hidden_states: torch.Tensor, + topk_output, + quant_info: FlashInferTrtllmFp8MoeQuantInfo, + runner_config: MoeRunnerConfig, + use_routed_topk: bool, +) -> torch.Tensor: + from sglang.srt.layers.moe.topk import TopKOutputChecker + from sglang.srt.layers.moe.utils import RoutingMethodType + + assert quant_info.w13_input_scale is not None + assert quant_info.output1_scales_scalar is not None + assert quant_info.output1_scales_gate_scalar is not None + assert quant_info.output2_scales_scalar is not None + a_q, _ = scaled_fp8_quant(hidden_states, quant_info.w13_input_scale) + with use_symmetric_memory( + get_parallel().tp_group, disabled=not is_allocation_symmetric() + ): + output = torch.empty( + hidden_states.shape[0], + hidden_states.shape[1], + dtype=torch.bfloat16, + device=hidden_states.device, + ) + + common = dict( + hidden_states=a_q, + gemm1_weights=quant_info.w13_weight, + output1_scales_scalar=quant_info.output1_scales_scalar, + output1_scales_gate_scalar=quant_info.output1_scales_gate_scalar, + gemm2_weights=quant_info.w2_weight, + output2_scales_scalar=quant_info.output2_scales_scalar, + num_experts=quant_info.global_num_experts, + intermediate_size=int(quant_info.w2_weight.shape[2]), + local_expert_offset=quant_info.local_expert_offset, + local_num_experts=quant_info.local_num_experts, + routed_scaling_factor=( + runner_config.routed_scaling_factor + if runner_config.routed_scaling_factor is not None + else 1.0 + ), + tune_max_num_tokens=next_power_of_2(a_q.shape[0]), + activation_type=quant_info.activation_type, + ) + if use_routed_topk: + routing = _get_routing_for_flashinfer_routed( + topk_output, promote_half_scores=True + ) + ids, weights = routing if isinstance(routing, tuple) else (routing, None) + trtllm_fp8_per_tensor_scale_routed_moe_out_wrapper( + topk_ids=ids, + topk_weights=weights, + output=output, + top_k=_routing_top_k(routing), + # Llama4's native FP8 body also consumes route scores as token scales. + routing_method_type=( + RoutingMethodType.Llama4 + if quant_info.routing_method_type == RoutingMethodType.Llama4 + else RoutingMethodType.TopK + ), + **common, + ) + else: + assert TopKOutputChecker.format_is_bypassed(topk_output) + config = topk_output.topk_config + result = trtllm_fp8_per_tensor_scale_moe_wrapper( + routing_logits=topk_output.router_logits.to(torch.bfloat16), + routing_bias=config.correction_bias, + top_k=config.top_k, + n_group=config.num_expert_group, + topk_group=config.topk_group, + use_routing_scales_on_input=False, + routing_method_type=quant_info.routing_method_type, + **common, + ) + output.copy_(result) + return output + + def fused_experts_none_to_flashinfer_trtllm_fp8( dispatch_output: StandardDispatchOutput | FlashinferDispatchOutput, quant_info: FlashInferTrtllmFp8MoeQuantInfo, @@ -800,6 +885,11 @@ def fused_experts_none_to_flashinfer_trtllm_fp8( else hidden_states.dtype ) topk_output = dispatch_output.topk_output + if TopKOutputChecker.format_is_bypassed(topk_output) and ( + topk_output.num_token_non_padded is not None + or topk_output.expert_location_dispatch_info is not None + ): + topk_output = topk_output.to_standard() materialized_topk = not use_routed_topk and TopKOutputChecker.format_is_standard( topk_output ) @@ -823,15 +913,8 @@ def fused_experts_none_to_flashinfer_trtllm_fp8( ) use_shuffled_weight = quant_info.use_mxfp8 defer_finalize = _deferred_finalize_enabled.get() - if defer_finalize and ( - not quant_info.block_quant - or use_routed_topk - or not TopKOutputChecker.format_is_bypassed(topk_output) - ): - raise RuntimeError( - "FP8 deferred finalize requires block quantization, the logits-based " - "FlashInfer TRTLLM backend, and bypassed TopK" - ) + if defer_finalize and not quant_info.block_quant: + raise RuntimeError("FP8 deferred finalize requires block quantization") if quant_info.block_quant: assert quant_info.weight_block_k is not None @@ -884,17 +967,15 @@ def fused_experts_none_to_flashinfer_trtllm_fp8( # during torch.compile for piecewise cuda graph. # Use custom op wrapper for torch.compile compatibility. if use_routed_topk: - assert runner_config.top_k is not None, ( - "runner_config.top_k is required for flashinfer_trtllm_routed." + routing = _get_routing_for_flashinfer_routed( + topk_output, promote_half_scores=True ) - routing = _get_routing_for_flashinfer_routed(topk_output) + routed_top_k = _routing_top_k(routing) topk_ids, topk_weights = ( routing if isinstance(routing, tuple) else (routing, None) ) - trtllm_fp8_block_scale_routed_moe_out_wrapper( - topk_ids=topk_ids, - topk_weights=topk_weights, + routed_kwargs = dict( routing_bias=None, hidden_states=a_q, hidden_states_scale=a_sf_t, @@ -906,7 +987,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp8( gemm2_weights=quant_info.w2_weight, gemm2_weights_scale=quant_info.w2_weight_scale_inv, num_experts=quant_info.global_num_experts, - top_k=runner_config.top_k, + top_k=routed_top_k, n_group=None, topk_group=None, intermediate_size=quant_info.intermediate_size, @@ -923,12 +1004,38 @@ def fused_experts_none_to_flashinfer_trtllm_fp8( else routing_method_type ), use_shuffled_weight=use_shuffled_weight, - output=symm_output, tune_max_num_tokens=next_power_of_2(a_q.shape[0]), - fp8_quantization_type=int(fp8_quantization_type), - activation_type=quant_info.activation_type, ) - output = cast(torch.Tensor, symm_output) + if defer_finalize: + from flashinfer.fused_moe import trtllm_fp8_block_scale_routed_moe + + deferred_kwargs = dict( + **routed_kwargs, + topk_ids=routing, + do_finalize=False, + fp8_quantization_type=fp8_quantization_type, + enable_pdl=a_q.shape[0] <= _TRTLLM_MOE_PDL_MAX_TOKENS, + ) + if quant_info.activation_type is not None: + from flashinfer.fused_moe.core import ActivationType + + deferred_kwargs["activation_type"] = ActivationType( + quant_info.activation_type + ) + output = _make_deferred_finalize_output( + trtllm_fp8_block_scale_routed_moe(**deferred_kwargs), + top_k=routed_top_k, + ) + else: + trtllm_fp8_block_scale_routed_moe_out_wrapper( + **routed_kwargs, + topk_ids=topk_ids, + topk_weights=topk_weights, + output=symm_output, + fp8_quantization_type=int(fp8_quantization_type), + activation_type=quant_info.activation_type, + ) + output = cast(torch.Tensor, symm_output) else: assert TopKOutputChecker.format_is_bypassed(topk_output) common_kwargs = dict( @@ -987,59 +1094,9 @@ def fused_experts_none_to_flashinfer_trtllm_fp8( ) output = cast(torch.Tensor, symm_output) else: - assert TopKOutputChecker.format_is_bypassed(topk_output) - assert quant_info.w13_input_scale is not None - assert quant_info.output1_scales_scalar is not None - assert quant_info.output1_scales_gate_scalar is not None - assert quant_info.output2_scales_scalar is not None - - a_q, _ = scaled_fp8_quant(hidden_states, quant_info.w13_input_scale) - - # Allocate output inside symmetric memory context - with use_symmetric_memory( - get_parallel().tp_group, disabled=not is_allocation_symmetric() - ): - symm_output = torch.empty( - hidden_states.shape[0], - hidden_states.shape[1], - dtype=torch.bfloat16, - device=hidden_states.device, - ) - - # Move kernel call outside context manager to avoid graph breaks - # during torch.compile for piecewise cuda graph. - # Use custom op wrapper for torch.compile compatibility. - - router_logits = router_logits.to(torch.bfloat16) - - output = trtllm_fp8_per_tensor_scale_moe_wrapper( - routing_logits=router_logits, - routing_bias=correction_bias, - hidden_states=a_q, - gemm1_weights=quant_info.w13_weight, - output1_scales_scalar=quant_info.output1_scales_scalar, - output1_scales_gate_scalar=quant_info.output1_scales_gate_scalar, - gemm2_weights=quant_info.w2_weight, - output2_scales_scalar=quant_info.output2_scales_scalar, - num_experts=quant_info.global_num_experts, - top_k=topk_config.top_k, - n_group=topk_config.num_expert_group, - topk_group=topk_config.topk_group, - intermediate_size=int(quant_info.w2_weight.shape[2]), - local_expert_offset=quant_info.local_expert_offset, - local_num_experts=quant_info.local_num_experts, - routed_scaling_factor=( - runner_config.routed_scaling_factor - if runner_config.routed_scaling_factor is not None - else 1.0 - ), - use_routing_scales_on_input=False, - routing_method_type=routing_method_type, - tune_max_num_tokens=next_power_of_2(a_q.shape[0]), - activation_type=quant_info.activation_type, + output = _run_fp8_per_tensor_scale_moe( + hidden_states, topk_output, quant_info, runner_config, use_routed_topk ) - symm_output.copy_(output) - output = symm_output return StandardCombineInput(hidden_states=output) @@ -1534,7 +1591,6 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( result = _make_deferred_finalize_output( result, top_k=_routing_top_k(routing), - materialized_routing=isinstance(routing, tuple), ) else: result = result[0] @@ -1724,7 +1780,6 @@ def fused_experts_none_to_flashinfer_trtllm_bf16( final_hidden_states = _make_deferred_finalize_output( final_hidden_states, top_k=runner_config.top_k, - materialized_routing=isinstance(routing, tuple), ) else: assert TopKOutputChecker.format_is_bypassed(topk_output) diff --git a/python/sglang/srt/layers/quantization/base_config.py b/python/sglang/srt/layers/quantization/base_config.py index b81b37ecb0..671dd6c7e9 100644 --- a/python/sglang/srt/layers/quantization/base_config.py +++ b/python/sglang/srt/layers/quantization/base_config.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: from sglang.srt.layers.moe.moe_runner.base import MoeQuantInfo from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput - from sglang.srt.layers.moe.utils import MoeRunnerBackendLike + from sglang.srt.layers.moe.utils import MoeRunnerBackendLike, RoutingMethodType from sglang.srt.models.utils import WeightsMapper @@ -123,6 +123,12 @@ class FusedMoEMethodBase(QuantizeMethodBase): """ return self.runner.runner_backend if self.runner is not None else None + def get_moe_routing_method_type( + self, layer: torch.nn.Module, requested_method: RoutingMethodType | None + ) -> RoutingMethodType | None: + """Return the routing policy consumed by the selected execution path.""" + return requested_method + @abstractmethod def apply( self, diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index bf1d66c57c..7d73ffcbea 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -2978,6 +2978,18 @@ class Fp8MoEMethod(FusedMoEMethodBase): return self.runner.runner_backend return self._moe_runner_backend + def get_moe_routing_method_type(self, layer, requested_method): + backend = self.get_moe_runner_backend(layer) + if ( + requested_method is None + and backend is not None + and ( + backend.is_flashinfer_trtllm() or backend.is_flashinfer_trtllm_routed() + ) + ): + return RoutingMethodType.DeepSeekV3 + return requested_method + def get_triton_quant_info(self, layer: torch.nn.Module) -> TritonMoeQuantInfo: use_rocm_mxfp8 = self.use_mxfp8 and _is_hip and _is_gfx95_supported return TritonMoeQuantInfo( @@ -3270,10 +3282,6 @@ class Fp8MoEMethod(FusedMoEMethodBase): is_gated=self.moe_runner_config.is_gated, ) - routing_method_type = layer.routing_method_type - if routing_method_type is None: - routing_method_type = RoutingMethodType.DeepSeekV3 - quant_info = FlashInferTrtllmFp8MoeQuantInfo( w13_weight=layer.w13_weight, w2_weight=layer.w2_weight, @@ -3281,7 +3289,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): local_expert_offset=moe_ep_rank * num_local_experts, local_num_experts=num_local_experts, intermediate_size=layer.w2_weight.shape[2], - routing_method_type=int(routing_method_type), + routing_method_type=int(layer.routing_method_type), block_quant=self.block_quant, use_mxfp8=getattr(self.quant_config, "use_mxfp8", False), weight_block_k=( diff --git a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_cutlass_moe.py b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_cutlass_moe.py index 96300858ac..7a1c5a9512 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_cutlass_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_cutlass_moe.py @@ -24,6 +24,7 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput + from sglang.srt.layers.moe.utils import RoutingMethodType # MXFP4 group/block size (E8M0 scale per 32 fp4 weights). _GROUP_SIZE = 32 @@ -127,6 +128,11 @@ class Mxfp4FlashinferCutlassMoEMethod: def get_moe_runner_backend(self, layer): return self.runner.runner_backend + def get_moe_routing_method_type( + self, layer: Module, requested_method: RoutingMethodType | None + ) -> RoutingMethodType | None: + return requested_method + def process_weights_after_loading(self, layer: Module) -> None: # Preserve the base FP4 post-load handling. self._fp8.process_weights_after_loading(layer) diff --git a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py index 801f2fca9d..523e4a9001 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py @@ -168,6 +168,11 @@ class Mxfp4FlashinferTrtllmMoEMethod: def get_moe_runner_backend(self, layer): return MoeRunnerBackend.FLASHINFER_MXFP4 + def get_moe_routing_method_type( + self, layer: Module, requested_method: RoutingMethodType | None + ) -> RoutingMethodType | None: + return requested_method + def create_weights( self, layer, diff --git a/python/sglang/srt/layers/quantization/mxfp4_humming_moe.py b/python/sglang/srt/layers/quantization/mxfp4_humming_moe.py index e34a1acd7a..bcc817227f 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_humming_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_humming_moe.py @@ -11,6 +11,7 @@ from sglang.srt.utils import log_info_on_rank0 if TYPE_CHECKING: from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput + from sglang.srt.layers.moe.utils import RoutingMethodType logger = logging.getLogger(__name__) @@ -38,6 +39,11 @@ class Mxfp4HummingMoEMethod: def get_moe_runner_backend(self, layer): return self.runner.runner_backend + def get_moe_routing_method_type( + self, layer: Module, requested_method: RoutingMethodType | None + ) -> RoutingMethodType | None: + return requested_method + def create_weights( self, layer: Module, diff --git a/python/sglang/srt/layers/quantization/mxfp4_marlin_moe.py b/python/sglang/srt/layers/quantization/mxfp4_marlin_moe.py index 43450d96dc..8ee5a96e23 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_marlin_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_marlin_moe.py @@ -13,6 +13,7 @@ from sglang.srt.utils import log_info_on_rank0, round_up, set_weight_attrs if TYPE_CHECKING: from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput + from sglang.srt.layers.moe.utils import RoutingMethodType logger = logging.getLogger(__name__) @@ -59,6 +60,11 @@ class Mxfp4MarlinMoEMethod: def get_moe_runner_backend(self, layer): return self.runner.runner_backend + def get_moe_routing_method_type( + self, layer: Module, requested_method: RoutingMethodType | None + ) -> RoutingMethodType | None: + return requested_method + def create_weights( self, layer: Module, diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 8a2173e6dc..b06ce6f963 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -312,7 +312,10 @@ class BailingMoE(nn.Module): self.topk.topk_config.apply_routed_scaling_factor_on_output = ( self.experts.should_fuse_routed_scaling_factor_in_topk ) - self.topk.bind_runner_backend(self.experts.runner_backend) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) if self.num_shared_experts > 0: intermediate_size = self.intermediate_size * self.num_shared_experts diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 7dfcaa34ab..2226ee096c 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -597,7 +597,10 @@ class BailingMoE(nn.Module): and getattr(quant_config, "is_fp4_experts", False) ), ) - self.topk.bind_runner_backend(self.experts.runner_backend) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) # Whether to apply routed_scaling_factor at model layer. # For A2A MoE paths (e.g., DeepEP), the runner/post_permute does not apply it, diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index 2ea986414f..b72f1f0de8 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -682,7 +682,10 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): self.topk.topk_config.apply_routed_scaling_factor_on_output = ( _is_npu or self.experts.should_fuse_routed_scaling_factor_in_topk ) - self.topk.bind_runner_backend(self.experts.runner_backend) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) if self.use_block_routing: _require_block_routing_runner_compatibility(self.experts, self.top_k) # shared expert diff --git a/test/registered/kernels/benchmark/moe/bench_shuffle_weighted_sum.py b/test/registered/kernels/benchmark/moe/bench_shuffle_weighted_sum.py new file mode 100644 index 0000000000..e74404b140 --- /dev/null +++ b/test/registered/kernels/benchmark/moe/bench_shuffle_weighted_sum.py @@ -0,0 +1,37 @@ +"""Whole mapped reduction, with cold-L2 graph input/output rotation.""" + +import torch + +from sglang.kernels.jit.benchmark import marker +from sglang.kernels.ops.moe.shuffle_weighted_sum import ( + _shuffle_weighted_sum_triton, + shuffle_weighted_sum, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=12, stage="nightly", runner_config="1-gpu-large") + + +@marker.parametrize("tokens,width,topk", [(32, 256, 2), (512, 256, 2), (64, 4096, 8)]) +@marker.benchmark("implementation", ["cuda", "triton"]) +def benchmark(tokens: int, width: int, topk: int, implementation: str): + values = torch.randn((tokens * topk, width), device="cuda", dtype=torch.bfloat16) + output = torch.empty((tokens, width), device="cuda", dtype=torch.bfloat16) + mapping = torch.randperm(tokens * topk, device="cuda", dtype=torch.int32) + weights = torch.rand((tokens, topk), device="cuda", dtype=torch.float32) + function = ( + shuffle_weighted_sum + if implementation == "cuda" + else _shuffle_weighted_sum_triton + ) + return marker.do_bench( + function, + input_args=(values, output, mapping, weights), + memory_args=(values, mapping, weights), + memory_output=(output,), + graph_clone_args=(0, 1, 2, 3), + ) + + +if __name__ == "__main__": + benchmark.run() diff --git a/test/registered/kernels/ops/moe/test_cutlass_fp8_inactive_routes.py b/test/registered/kernels/ops/moe/test_cutlass_fp8_inactive_routes.py index d21a8a8395..3115ab3dc4 100644 --- a/test/registered/kernels/ops/moe/test_cutlass_fp8_inactive_routes.py +++ b/test/registered/kernels/ops/moe/test_cutlass_fp8_inactive_routes.py @@ -6,8 +6,10 @@ from unittest.mock import patch import torch +from sglang.kernels.ops.moe.cutlass_prepare_moe_input import cutlass_prepare_moe_input from sglang.kernels.ops.moe.shuffle_weighted_sum import ( _shuffle_weighted_sum_kernel, + _shuffle_weighted_sum_triton, shuffle_weighted_sum, ) from sglang.srt.layers.moe import cutlass_moe @@ -15,10 +17,317 @@ from sglang.srt.layers.moe.cutlass_moe_params import CutlassMoEParams, CutlassMo from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=12, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci(est_time=180, stage="base-b-kernel-unit", runner_config="1-gpu-large") class TestCutlassFp8InactiveRoutes(CustomTestCase): + def test_prepare_completes_maps_across_route_changes(self): + """Every live route is gathered once; inactive and unused capacity is defined.""" + experts, tokens, topk, n, k = 3, 265, 2, 96, 160 + patterns = ( + [0, 1, 2, -1, -7, experts, 1, 2, 0, 0], + [-1, -9, experts, 99], + [0, 1, 2, 0, 1, 2], + ) + states = [ + torch.tensor( + (pattern * ((tokens * topk + len(pattern) - 1) // len(pattern)))[ + : tokens * topk + ], + device="cuda", + dtype=torch.int32, + ).view(tokens, topk) + for pattern in patterns + ] + ids = states[0].clone() + offsets = torch.full((experts + 1,), -999, device="cuda", dtype=torch.int32) + sizes1 = torch.full((experts, 3), -999, device="cuda", dtype=torch.int32) + sizes2 = torch.full_like(sizes1, -999) + a_map = torch.full((ids.numel(),), -999, device="cuda", dtype=torch.int32) + c_map = torch.full_like(a_map, -999) + block_offsets = torch.full_like(offsets, -999) + + def prepare(blocks): + cutlass_prepare_moe_input( + ids, offsets, sizes1, sizes2, a_map, c_map, experts, n, k, blocks + ) + + def check(): + flat_ids = ids.flatten() + live_mask = (flat_ids >= 0) & (flat_ids < experts) + counts = torch.bincount(flat_ids[live_mask], minlength=experts) + expected_offsets = torch.cat((counts.new_zeros(1), counts.cumsum(0))) + torch.testing.assert_close(offsets, expected_offsets.int(), rtol=0, atol=0) + torch.testing.assert_close( + sizes1, + torch.stack( + ( + counts, + counts.new_full((experts,), 2 * n), + counts.new_full((experts,), k), + ), + dim=1, + ).int(), + rtol=0, + atol=0, + ) + torch.testing.assert_close( + sizes2, + torch.stack( + ( + counts, + counts.new_full((experts,), k), + counts.new_full((experts,), n), + ), + dim=1, + ).int(), + rtol=0, + atol=0, + ) + live = int(expected_offsets[-1]) + torch.testing.assert_close( + c_map[live_mask].sort().values, + torch.arange(live, device="cuda", dtype=torch.int32), + rtol=0, + atol=0, + ) + self.assertTrue(bool((c_map[~live_mask] == -1).all())) + routes = torch.arange(ids.numel(), device="cuda")[live_mask] + torch.testing.assert_close( + a_map[c_map[live_mask].long()], (routes // topk).int(), rtol=0, atol=0 + ) + self.assertTrue(bool((a_map[live:] == 0).all())) + for expert in range(experts): + destinations = c_map[flat_ids == expert] + self.assertTrue(bool((destinations >= offsets[expert]).all())) + self.assertTrue(bool((destinations < offsets[expert + 1]).all())) + return counts + + for blocks in (None, block_offsets): + with self.subTest(blockscale=blocks is not None): + prepare(blocks) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + prepare(blocks) + for state in (*states, states[0]): + ids.copy_(state) + a_map.fill_(-999) + c_map.fill_(-999) + offsets.fill_(-999) + sizes1.fill_(-999) + sizes2.fill_(-999) + if blocks is not None: + blocks.fill_(-999) + graph.replay() + counts = check() + if blocks is not None: + expected = torch.cat( + ( + counts.new_zeros(1), + (((counts + 127) // 128) * 128).cumsum(0), + ) + ) + torch.testing.assert_close( + blocks, expected.int(), rtol=0, atol=0 + ) + + for shape in ((0, topk), (tokens, 0)): + with self.subTest(shape=shape): + empty_ids = torch.empty(shape, device="cuda", dtype=torch.int32) + empty_map = torch.empty((0,), device="cuda", dtype=torch.int32) + cutlass_prepare_moe_input( + empty_ids, + offsets, + sizes1, + sizes2, + empty_map, + empty_map, + experts, + n, + k, + block_offsets, + ) + self.assertEqual(int(offsets.count_nonzero()), 0) + self.assertEqual(int(block_offsets.count_nonzero()), 0) + self.assertEqual(int(sizes1[:, 0].count_nonzero()), 0) + self.assertEqual(int(sizes2[:, 0].count_nonzero()), 0) + + def test_prepare_has_no_expert_sized_global_scratch(self): + """A warmed producer needs no extra storage even when experts exceed routes.""" + experts, tokens, topk = 513, 2, 2 + ids = torch.tensor( + [[0, experts - 1], [-1, experts]], device="cuda", dtype=torch.int32 + ) + offsets = torch.empty(experts + 1, device="cuda", dtype=torch.int32) + sizes1 = torch.empty((experts, 3), device="cuda", dtype=torch.int32) + sizes2 = torch.empty_like(sizes1) + a_map = torch.empty(tokens * topk, device="cuda", dtype=torch.int32) + c_map = torch.empty_like(a_map) + + def prepare(): + cutlass_prepare_moe_input( + ids, offsets, sizes1, sizes2, a_map, c_map, experts, 96, 160 + ) + + for _ in range(3): + prepare() + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + allocated = torch.cuda.memory_allocated() + prepare() + torch.cuda.synchronize() + self.assertEqual(torch.cuda.max_memory_allocated(), allocated) + + def test_prepare_transposed_shapes_match_native_w4a8(self): + """W4A8 metadata and complete maps survive live/inactive graph transitions.""" + from sgl_kernel import get_cutlass_w4a8_moe_mm_data + + experts, n, k = 17, 96, 160 + patterns = ( + [[0, 16], [1, 0], [16, 1], [0, 16]], + [[0, 16], [1, -1], [experts, -7], [16, experts + 3]], + [[-1, experts], [-7, experts + 1], [-1, -9], [experts, experts]], + ) + states = [ + torch.tensor(pattern, device="cuda", dtype=torch.int32) + for pattern in patterns + ] + ids = states[0].clone() + offsets = torch.full((experts + 1,), -999, device="cuda", dtype=torch.int32) + sizes1 = torch.full((experts, 3), -999, device="cuda", dtype=torch.int32) + sizes2 = torch.full_like(sizes1, -999) + a_map = torch.full((ids.numel(),), -999, device="cuda", dtype=torch.int32) + c_map = torch.full_like(a_map, -999) + native_offsets = offsets.clone() + native_sizes1 = sizes1.clone() + native_sizes2 = sizes2.clone() + + def prepare(): + cutlass_prepare_moe_input( + ids, + offsets, + sizes1, + sizes2, + a_map, + c_map, + experts, + n, + k, + transpose_problem_sizes=True, + ) + + prepare() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + prepare() + for state in (*states, states[0]): + with self.subTest(ids=state.tolist()): + ids.copy_(state) + for value in (offsets, sizes1, sizes2, a_map, c_map): + value.fill_(-999) + graph.replay() + # The installed native producer supplies the integer W4A8 ABI oracle. + # It does not read or write a_map/c_map, and does not define their order. + get_cutlass_w4a8_moe_mm_data( + ids, + native_offsets, + native_sizes1, + native_sizes2, + a_map, + c_map, + experts, + n, + k, + ) + for actual, reference in ( + (offsets, native_offsets), + (sizes1, native_sizes1), + (sizes2, native_sizes2), + ): + torch.testing.assert_close(actual, reference, rtol=0, atol=0) + flat = ids.flatten() + local = (flat >= 0) & (flat < experts) + live = int(offsets[-1]) + torch.testing.assert_close( + c_map[local].sort().values, + torch.arange(live, device="cuda", dtype=torch.int32), + rtol=0, + atol=0, + ) + self.assertTrue(bool((c_map[~local] == -1).all())) + for expert in range(experts): + rows = c_map[flat == expert] + self.assertTrue(bool((rows >= offsets[expert]).all())) + self.assertTrue(bool((rows < offsets[expert + 1]).all())) + routes = torch.arange(ids.numel(), device="cuda")[local] + torch.testing.assert_close( + a_map[c_map[local].long()], + (routes // ids.shape[1]).int(), + rtol=0, + atol=0, + ) + self.assertTrue(bool((a_map[live:] == 0).all())) + torch.testing.assert_close(ids, state, rtol=0, atol=0) + + def prepare_and_consume(): + prepare() + return offsets + 1 + + compiled = torch.compile(prepare_and_consume, fullgraph=True) + offsets.fill_(-999) + sizes1.fill_(-999) + sizes2.fill_(-999) + result = compiled() + torch.testing.assert_close(result, native_offsets + 1, rtol=0, atol=0) + torch.testing.assert_close(sizes1, native_sizes1, rtol=0, atol=0) + torch.testing.assert_close(sizes2, native_sizes2, rtol=0, atol=0) + + from torch._subclasses.fake_tensor import FakeTensorMode + + with FakeTensorMode(): + fake_ids = torch.empty(ids.shape, device="cuda", dtype=torch.int32) + fake_offsets = torch.empty(experts + 1, device="cuda", dtype=torch.int32) + fake_sizes1 = torch.empty((experts, 3), device="cuda", dtype=torch.int32) + fake_sizes2 = torch.empty_like(fake_sizes1) + fake_a_map = torch.empty(ids.numel(), device="cuda", dtype=torch.int32) + fake_c_map = torch.empty_like(fake_a_map) + self.assertIsNone( + cutlass_prepare_moe_input( + fake_ids, + fake_offsets, + fake_sizes1, + fake_sizes2, + fake_a_map, + fake_c_map, + experts, + n, + k, + transpose_problem_sizes=True, + ) + ) + + def test_prepare_rejects_noncontiguous_metadata(self): + """Reject layouts whose physical order differs from the route-map contract.""" + ids = torch.tensor([[0, 1], [1, -1], [2, 0]], device="cuda", dtype=torch.int32) + offsets = torch.empty(4, device="cuda", dtype=torch.int32) + sizes1 = torch.empty((3, 3), device="cuda", dtype=torch.int32) + sizes2 = torch.empty_like(sizes1) + a_map = torch.empty(ids.numel(), device="cuda", dtype=torch.int32) + c_map = torch.empty_like(a_map) + block_offsets = torch.empty_like(offsets) + args = [ids, offsets, sizes1, sizes2, a_map, c_map, 3, 96, 160, block_offsets] + for index in (0, 1, 2, 3, 4, 5, 9): + value = args[index] + strided = torch.empty((*value.shape, 2), device="cuda", dtype=torch.int32)[ + ..., 0 + ] + strided.copy_(value) + with self.subTest(argument=index): + invalid_args = args.copy() + invalid_args[index] = strided + with self.assertRaisesRegex((RuntimeError, ValueError), "contiguous"): + cutlass_prepare_moe_input(*invalid_args) + def test_masked_reduction_ignores_poisoned_slots(self): for dtype in (torch.float16, torch.bfloat16): with self.subTest(dtype=dtype): @@ -90,7 +399,9 @@ class TestCutlassFp8InactiveRoutes(CustomTestCase): output = torch.empty( (tokens, 257), device="cuda", dtype=dtype ) - shuffle_weighted_sum(inputs, output, mapping, weights) + _shuffle_weighted_sum_triton( + inputs, output, mapping, weights + ) expected = ( torch.tensor([3.5, 2, 0], device="cuda", dtype=dtype) .repeat(copies)[:tokens, None] @@ -99,17 +410,530 @@ class TestCutlassFp8InactiveRoutes(CustomTestCase): torch.testing.assert_close(output, expected, rtol=0, atol=0) if specialization_count is None: specialization_count = len(cache) + self.assertGreater(specialization_count, 0) self.assertEqual(len(cache), specialization_count) mapping.fill_(-1) inputs.fill_(float("nan")) weights.fill_(float("nan")) output.fill_(float("nan")) - shuffle_weighted_sum(inputs, output, mapping, weights) + _shuffle_weighted_sum_triton( + inputs, output, mapping, weights + ) self.assertTrue(bool(torch.isfinite(output).all())) self.assertEqual(int(torch.count_nonzero(output)), 0) self.assertEqual(len(cache), specialization_count) + def test_cuda_reduction_matches_triton_for_ragged_strided_views(self): + """Guard masked gathers and exact slot arithmetic across dtype/layout keys.""" + for dtype in (torch.float16, torch.bfloat16): + for weight_dtype in (torch.float16, torch.bfloat16, torch.float32): + for index_dtype in (torch.int32, torch.int64): + with self.subTest( + dtype=dtype, weights=weight_dtype, mapping=index_dtype + ): + tokens, rows, width, topk = 5, 6, 257, 3 + inputs_storage = torch.full( + (rows + 2, width + 7), + float("nan"), + device="cuda", + dtype=dtype, + ) + inputs = inputs_storage[1 : rows + 1, 3 : width + 3] + inputs[:4].copy_( + ( + torch.arange(4 * width, device="cuda").view(4, width) + % 29 + - 14 + ).to(dtype) + / 16 + ) + invalid = -(2**40) if index_dtype == torch.int64 else -999 + invalid_positive = ( + 2**40 if index_dtype == torch.int64 else 2**31 - 1 + ) + mapping_storage = torch.full( + (tokens * topk + 4,), + invalid, + device="cuda", + dtype=index_dtype, + ) + mapping = mapping_storage[2:-2].view(tokens, topk) + mapping.copy_( + torch.tensor( + [ + [0, 1, 2], + [-1, 3, rows], + [invalid, invalid_positive, -1], + [3, 0, -1], + [1, 2, 0], + ], + device="cuda", + dtype=index_dtype, + ) + ) + score_storage = torch.full( + (tokens * 2 + 1, topk * 3 + 2), + float("nan"), + device="cuda", + dtype=weight_dtype, + ) + weights = score_storage[ + 1 : tokens * 2 + 1 : 2, 1 : topk * 3 + 1 : 3 + ] + weights.copy_( + torch.tensor( + [ + [0.25, 0.5, -0.125], + [float("nan"), 0.75, float("nan")], + [float("nan")] * 3, + [0.5, -0.25, float("nan")], + [0.125, -0.5, 0.25], + ], + device="cuda", + dtype=weight_dtype, + ) + ) + out_storage = torch.full( + (tokens + 2, width + 9), -17, device="cuda", dtype=dtype + ) + ref_storage = out_storage.clone() + output = out_storage[1 : tokens + 1, 4 : width + 4] + reference = ref_storage[1 : tokens + 1, 4 : width + 4] + saved = [ + value.clone() + for value in ( + inputs_storage, + mapping_storage, + score_storage, + ) + ] + _shuffle_weighted_sum_triton( + inputs, reference, mapping, weights + ) + shuffle_weighted_sum(inputs, output, mapping, weights) + torch.testing.assert_close( + out_storage.view(torch.int16), + ref_storage.view(torch.int16), + rtol=0, + atol=0, + ) + for value, before in zip( + (inputs_storage, mapping_storage, score_storage), saved + ): + torch.testing.assert_close( + value, before, rtol=0, atol=0, equal_nan=True + ) + self.assertEqual(int(torch.count_nonzero(output[2])), 0) + + def test_cuda_reduction_preserves_fma_denormals_and_slot_order(self): + """FMA residuals and BF16 subnormal operands must survive the CUDA port.""" + for dtype, value in ((torch.float16, 1025.0), (torch.bfloat16, 129.0)): + with self.subTest(dtype=dtype): + inputs = torch.tensor( + [[-value], [value], [65504.0], [-65504.0], [2**-14]], + device="cuda", + dtype=dtype, + ) + mapping = torch.tensor( + [[0, 1, -1], [2, 3, 4]], device="cuda", dtype=torch.int64 + ) + weights = torch.tensor( + [[1.0, 1.0 - 2**-23, float("nan")], [1.0, 1.0, 1.0]], + device="cuda", + dtype=torch.float32, + ) + output = torch.empty((2, 1), device="cuda", dtype=dtype) + reference = torch.empty_like(output) + _shuffle_weighted_sum_triton(inputs, reference, mapping, weights) + shuffle_weighted_sum(inputs, output, mapping, weights) + expected = torch.tensor( + [[-value * 2**-23], [2**-14]], device="cuda", dtype=dtype + ) + torch.testing.assert_close( + output.view(torch.int16), expected.view(torch.int16), rtol=0, atol=0 + ) + torch.testing.assert_close( + output.view(torch.int16), + reference.view(torch.int16), + rtol=0, + atol=0, + ) + inputs = torch.tensor( + [[2**-133, -(2**-133), -(2**-133)]], device="cuda", dtype=torch.bfloat16 + ) + mapping = torch.tensor([[0, -1]], device="cuda", dtype=torch.int32) + weights = torch.tensor( + [[2**120, float("nan")]], device="cuda", dtype=torch.float32 + ) + output = torch.empty((1, 3), device="cuda", dtype=torch.bfloat16) + shuffle_weighted_sum(inputs, output, mapping, weights) + expected = torch.tensor( + [[2**-13, -(2**-13), -(2**-13)]], device="cuda", dtype=torch.bfloat16 + ) + torch.testing.assert_close( + output.view(torch.int16), expected.view(torch.int16), rtol=0, atol=0 + ) + weights[0, 0] = 1.0 + shuffle_weighted_sum(inputs, output, mapping, weights) + torch.testing.assert_close( + output.view(torch.int16), inputs.view(torch.int16), rtol=0, atol=0 + ) + weights[0, 0] = 2**-40 + shuffle_weighted_sum(inputs, output, mapping, weights) + # FP32 underflow creates -0 in slot0; the inactive +0 FMA restores +0. + self.assertTrue(bool((output.view(torch.int16) == 0).all())) + + def test_cuda_reduction_graph_changes_and_empty_routes(self): + """A current-stream graph must rewrite poisoned output on every map change.""" + for dtype in (torch.float16, torch.bfloat16): + with self.subTest(dtype=dtype): + inputs = ( + torch.arange(4 * 513, device="cuda").view(4, 513).to(dtype) / 128 + ) + mapping = torch.tensor([0, 1, 2, 3], device="cuda", dtype=torch.int64) + weights = torch.tensor( + [[0.25, 0.75], [0.5, -0.25]], device="cuda", dtype=torch.float32 + ) + output = torch.empty((2, 513), device="cuda", dtype=dtype) + reference = torch.empty_like(output) + shuffle_weighted_sum(inputs, output, mapping, weights) + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=side): + shuffle_weighted_sum(inputs, output, mapping, weights) + torch.cuda.current_stream().wait_stream(side) + for state in ("partial", "zero", "recovery"): + inputs.copy_( + torch.arange(4 * 513, device="cuda").view(4, 513).to(dtype) + / 128 + ) + mapping.copy_( + torch.tensor([0, 1, 2, 3], device="cuda", dtype=torch.int64) + ) + weights.copy_( + torch.tensor([[0.25, 0.75], [0.5, -0.25]], device="cuda") + ) + if state == "partial": + mapping[0] = -1 + weights[0, 0] = float("nan") + inputs[0].fill_(float("nan")) + elif state == "zero": + mapping.fill_(-1) + weights.fill_(float("nan")) + inputs.fill_(float("nan")) + output.fill_(float("nan")) + _shuffle_weighted_sum_triton(inputs, reference, mapping, weights) + graph.replay() + torch.testing.assert_close( + output.view(torch.int16), + reference.view(torch.int16), + rtol=0, + atol=0, + ) + self.assertTrue(bool(torch.isfinite(output).all())) + for input_rows, topk in ((0, 2), (4, 0)): + empty_inputs = torch.empty( + (input_rows, 513), device="cuda", dtype=dtype + ) + empty_maps = torch.zeros( + (2 * topk,), device="cuda", dtype=torch.int32 + ) + empty_weights = torch.full( + (2, topk), float("nan"), device="cuda", dtype=torch.float32 + ) + output.fill_(float("nan")) + shuffle_weighted_sum( + empty_inputs, output, empty_maps, empty_weights + ) + self.assertTrue(bool((output.view(torch.int16) == 0).all())) + empty_output = output[:0] + shuffle_weighted_sum(inputs, empty_output, mapping[:0], weights[:0]) + shuffle_weighted_sum(inputs[:, :0], output[:, :0], mapping, weights) + # FP32 input/output retains the original Triton path and contract. + inputs = torch.tensor([[1.0, 2.0]], device="cuda") + output = torch.empty_like(inputs) + shuffle_weighted_sum( + inputs, + output, + torch.tensor([0], device="cuda"), + torch.tensor([[0.5]], device="cuda"), + ) + torch.testing.assert_close(output, inputs / 2, rtol=0, atol=0) + + def test_cuda_reduction_preserves_identity_alias_and_compiled_mutation(self): + """Preserve safe in-place topk1 and expose the mutation to compile/fake.""" + dtype = torch.bfloat16 + inputs = torch.arange(3 * 257, device="cuda").view(3, 257).to(dtype) / 128 + saved = inputs.clone() + mapping = torch.arange(3, device="cuda", dtype=torch.int32) + weights = torch.full((3, 1), 0.5, device="cuda") + with torch.no_grad(): + shuffle_weighted_sum(inputs, inputs, mapping, weights) + torch.testing.assert_close(inputs, saved / 2, rtol=0, atol=0) + storage = torch.cat((saved, torch.full_like(saved, float("nan")))) + with torch.no_grad(): + shuffle_weighted_sum(storage[:3], storage[3:], mapping, weights) + torch.testing.assert_close(storage[:3], saved, rtol=0, atol=0) + torch.testing.assert_close(storage[3:], saved / 2, rtol=0, atol=0) + + def reduce_and_consume(values, output, route_map, scores): + shuffle_weighted_sum(values, output, route_map, scores) + return output + 1 + + compiled = torch.compile(reduce_and_consume, fullgraph=True) + output = torch.empty_like(inputs) + reference = torch.empty_like(inputs) + for active in (True, False, True): + mapping.copy_(torch.arange(3, device="cuda", dtype=torch.int32)) + weights.fill_(0.5) + if not active: + mapping.fill_(-1) + weights.fill_(float("nan")) + _shuffle_weighted_sum_triton(inputs, reference, mapping, weights) + output.fill_(float("nan")) + with torch.no_grad(): + result = compiled(inputs, output, mapping, weights) + torch.testing.assert_close( + output.view(torch.int16), reference.view(torch.int16), rtol=0, atol=0 + ) + torch.testing.assert_close(result, reference + 1, rtol=0, atol=0) + from torch._subclasses.fake_tensor import FakeTensorMode + + with FakeTensorMode(): + fake_values = torch.empty((6, 257), dtype=dtype, device="cuda") + fake_output = torch.empty((3, 257), dtype=dtype, device="cuda") + fake_map = torch.empty(6, dtype=torch.int32, device="cuda") + fake_scores = torch.empty((3, 2), dtype=torch.float32, device="cuda") + self.assertIsNone( + shuffle_weighted_sum(fake_values, fake_output, fake_map, fake_scores) + ) + + def _assert_reduction_bits_equal(self, actual, expected): + torch.testing.assert_close(torch.isnan(actual), torch.isnan(expected)) + bit_dtype = torch.int32 if expected.dtype == torch.float32 else torch.int16 + finite_or_inf = ~torch.isnan(expected) + torch.testing.assert_close( + actual.view(bit_dtype)[finite_or_inf], + expected.view(bit_dtype)[finite_or_inf], + rtol=0, + atol=0, + ) + + def test_reduction_fuses_score_cast_to_input_dtype(self): + """The optional cast matches the previous explicit Torch conversion chain.""" + width, topk, rows = 17, 2, 5 + for dtype, value in ((torch.float16, 1025.0), (torch.bfloat16, 129.0)): + for weight_dtype in (torch.float16, torch.bfloat16, torch.float32): + with self.subTest(dtype=dtype, weight_dtype=weight_dtype): + input_storage = torch.full( + (rows, width + 3), float("nan"), device="cuda", dtype=dtype + ) + inputs = input_storage[:, :width] + inputs[0].fill_(-value) + inputs[1].fill_(value) + inputs[2].fill_(1) + inputs[3].fill_(2.0**120 if dtype == torch.bfloat16 else 1.0) + mapping = torch.tensor( + [[0, 1]] + + [[2, 2]] * 4 + + [[2, -1]] * 2 + + [[3, -1], [2, 2], [2, 2], [-1, rows + 5], [2, 2], [2, -1]], + device="cuda", + dtype=torch.int64, + ) + tokens = mapping.shape[0] + score_storage = torch.full( + (tokens * 2 + 1, topk * 3 + 2), + float("nan"), + device="cuda", + dtype=weight_dtype, + ) + weights = score_storage[ + 1 : 2 * tokens + 1 : 2, 1 : 3 * topk + 1 : 3 + ] + weights.copy_( + torch.tensor( + [ + [1.0, 1.0 - 2**-23], + [1.0 + 2**-11, -1.0], + [1.0 + 3 * 2**-11, -1.0], + [1.0 + 2**-8, -1.0], + [1.0 + 3 * 2**-8, -1.0], + [2**-24, float("nan")], + [2**-25, float("nan")], + [2**-133, float("nan")], + [float("inf"), -float("inf")], + [float("nan"), 1.0], + [float("nan"), float("inf")], + [-0.0, 0.0], + [0.5, float("nan")], + ], + device="cuda", + dtype=weight_dtype, + ) + ) + output = torch.empty((tokens, width), device="cuda", dtype=dtype) + reference = torch.empty_like(output) + triton_output = torch.empty_like(output) + saved = [x.clone() for x in (input_storage, mapping, score_storage)] + shuffle_weighted_sum( + inputs, reference, mapping, weights.to(inputs.dtype) + ) + shuffle_weighted_sum( + inputs, + output, + mapping, + weights, + cast_weights_to_input_dtype=True, + ) + _shuffle_weighted_sum_triton( + inputs, + triton_output, + mapping, + weights, + cast_weights_to_input_dtype=True, + ) + self._assert_reduction_bits_equal(output, reference) + self._assert_reduction_bits_equal(triton_output, reference) + self.assertTrue(bool((output[10].view(torch.int16) == 0).all())) + if weight_dtype == torch.float32: + uncast = torch.empty_like(output) + shuffle_weighted_sum(inputs, uncast, mapping, weights) + self.assertTrue(bool((output[0].view(torch.int16) == 0).all())) + self.assertTrue(bool((uncast[0] != output[0]).all())) + # A wider output must still round scores to the input dtype. + wide_output = output.float() + wide_reference = reference.float() + shuffle_weighted_sum( + inputs, + wide_reference, + mapping, + weights.to(inputs.dtype), + ) + shuffle_weighted_sum( + inputs, + wide_output, + mapping, + weights, + cast_weights_to_input_dtype=True, + ) + self._assert_reduction_bits_equal(wide_output, wide_reference) + for tensor, before in zip( + (input_storage, mapping, score_storage), saved + ): + torch.testing.assert_close( + tensor, before, rtol=0, atol=0, equal_nan=True + ) + for weight_dtype in (torch.float64, torch.int64): + with self.subTest(dtype=dtype, weight_dtype=weight_dtype): + inputs = torch.ones((1, width), device="cuda", dtype=dtype) + mapping = torch.tensor([[0, 0]], device="cuda", dtype=torch.int32) + weights = torch.tensor( + [[257, -256]], device="cuda", dtype=weight_dtype + ) + output = torch.empty_like(inputs) + reference = torch.empty_like(inputs) + shuffle_weighted_sum( + inputs, reference, mapping, weights.to(inputs.dtype) + ) + shuffle_weighted_sum( + inputs, + output, + mapping, + weights, + cast_weights_to_input_dtype=True, + ) + self._assert_reduction_bits_equal(output, reference) + + def test_reduction_score_cast_survives_graph_and_compile(self): + """Captured score casts follow live metadata without changing borrowed tensors.""" + width = 17 + for dtype in (torch.float16, torch.bfloat16): + with self.subTest(dtype=dtype): + value = 1025.0 if dtype == torch.float16 else 129.0 + initial = torch.tensor( + [[-value] * width, [value] * width, [1.0] * width, [2.0] * width], + device="cuda", + dtype=dtype, + ) + inputs = initial.clone() + mapping = torch.tensor( + [[0, 1], [2, 3]], device="cuda", dtype=torch.int64 + ) + scores = torch.full((2, 4), float("nan"), device="cuda") + weights = scores[:, ::2] + weights.copy_( + torch.tensor([[1.0, 1.0 - 2**-23], [0.25, 0.75]], device="cuda") + ) + output = torch.empty((2, width), device="cuda", dtype=dtype) + reference = torch.empty_like(output) + + def run(values, result, route_map, route_weights): + shuffle_weighted_sum( + values, + result, + route_map, + route_weights, + cast_weights_to_input_dtype=True, + ) + return result + 1 + + run(inputs, output, mapping, weights) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + shuffle_weighted_sum( + inputs, + output, + mapping, + weights, + cast_weights_to_input_dtype=True, + ) + for state in ("partial", "zero", "recovery"): + inputs.copy_(initial) + mapping.copy_(torch.tensor([[0, 1], [2, 3]], device="cuda")) + weights.copy_( + torch.tensor([[1.0, 1.0 - 2**-23], [0.25, 0.75]], device="cuda") + ) + if state == "partial": + mapping[1, 1] = -1 + inputs[3].fill_(float("nan")) + weights[1, 1] = float("nan") + elif state == "zero": + mapping.fill_(-1) + inputs.fill_(float("nan")) + weights.fill_(float("nan")) + saved = [x.clone() for x in (inputs, mapping, scores)] + shuffle_weighted_sum( + inputs, reference, mapping, weights.to(inputs.dtype) + ) + output.fill_(float("nan")) + graph.replay() + self._assert_reduction_bits_equal(output, reference) + for tensor, before in zip((inputs, mapping, scores), saved): + torch.testing.assert_close( + tensor, before, rtol=0, atol=0, equal_nan=True + ) + if dtype == torch.bfloat16: + compiled = torch.compile(run, fullgraph=True) + with torch.no_grad(): + result = compiled(inputs, output, mapping, weights) + self._assert_reduction_bits_equal(output, reference) + torch.testing.assert_close(result, reference + 1, rtol=0, atol=0) + from torch._subclasses.fake_tensor import FakeTensorMode + + with FakeTensorMode(): + self.assertIsNone( + shuffle_weighted_sum( + torch.empty((4, width), device="cuda", dtype=dtype), + torch.empty((2, width), device="cuda", dtype=dtype), + torch.empty((2, 2), device="cuda", dtype=torch.int64), + torch.empty((2, 2), device="cuda"), + cast_weights_to_input_dtype=True, + ) + ) + def test_native_fp8_padding_survives_graph_replay(self): if torch.cuda.get_device_capability()[0] not in (9, 10, 12): self.skipTest("CUTLASS block FP8 requires SM90, SM100 or SM120") @@ -222,8 +1046,36 @@ class TestCutlassFp8InactiveRoutes(CustomTestCase): enable_es=(mxfp8, mxfp8), ) - reference = run(inputs.nan_to_num(), ids.clamp_min(0), weights.nan_to_num()) - prepare = cutlass_moe.prepare_moe_input + from sgl_kernel import prepare_moe_input as legacy_prepare_moe_input + + valid_inputs = inputs.nan_to_num() + valid_ids = ids.clamp_min(0) + valid_weights = torch.full_like(weights, 1 / ids.size(1)) + # Both producers define every map entry when all routes are valid. + # Keep the same native entry and GEMM arguments so only map production + # changes; compare raw output bits rather than a numerical tolerance. + with patch.object( + cutlass_moe, "cutlass_prepare_moe_input", legacy_prepare_moe_input + ): + legacy_valid = run(valid_inputs, valid_ids, valid_weights) + legacy_repeat = run(valid_inputs, valid_ids, valid_weights) + torch.testing.assert_close( + legacy_valid.view(torch.int16), + legacy_repeat.view(torch.int16), + rtol=0, + atol=0, + msg="Legacy native provider is nondeterministic; exact producer parity is unresolved", + ) + actual_valid = run(valid_inputs, valid_ids, valid_weights) + torch.testing.assert_close( + actual_valid.view(torch.int16), + legacy_valid.view(torch.int16), + rtol=0, + atol=0, + ) + + reference = run(valid_inputs, valid_ids, weights.nan_to_num()) + prepare = cutlass_moe.cutlass_prepare_moe_input def checked_prepare(*args): prepare(*args) @@ -245,7 +1097,7 @@ class TestCutlassFp8InactiveRoutes(CustomTestCase): # The pre-fix map allocation fails checked_prepare before an unsafe gather. with ( patch.object(torch, "empty", poisoned_empty), - patch.object(cutlass_moe, "prepare_moe_input", checked_prepare), + patch.object(cutlass_moe, "cutlass_prepare_moe_input", checked_prepare), ): actual = run(inputs, ids, weights) torch.testing.assert_close(actual, reference, rtol=0.025, atol=0.01) @@ -269,6 +1121,12 @@ class TestCutlassFp8InactiveRoutes(CustomTestCase): self.assertTrue(bool(torch.isfinite(result).all())) self.assertEqual(int(torch.count_nonzero(result)), 0) + empty_result = run(inputs[:0], ids[:0], weights[:0]) + self.assertEqual(empty_result.shape, (0, hidden)) + no_route_result = run(inputs, ids[:, :0], weights[:, :0]) + self.assertTrue(bool(torch.isfinite(no_route_result).all())) + self.assertEqual(int(torch.count_nonzero(no_route_result)), 0) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/kernels/ops/moe/test_cutlass_w4a8_inactive_routes.py b/test/registered/kernels/ops/moe/test_cutlass_w4a8_inactive_routes.py index 3c05012742..75c490a73d 100644 --- a/test/registered/kernels/ops/moe/test_cutlass_w4a8_inactive_routes.py +++ b/test/registered/kernels/ops/moe/test_cutlass_w4a8_inactive_routes.py @@ -1,4 +1,4 @@ -"""W4A8 routing must pack inactive routes after live experts, including at EP1.""" +"""W4A8 packs only local routes, preserving raw dropped-route IDs at EP1 and EP2.""" import unittest from unittest.mock import patch @@ -6,6 +6,9 @@ from unittest.mock import patch import torch import torch.nn.functional as F +from sglang.kernels.ops.moe.ep_moe_kernels import ( + compute_cutlass_w4a8_moe_input_scale, +) from sglang.srt.layers.moe import cutlass_w4a8_moe as w4a8 from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cuda_ci @@ -20,15 +23,167 @@ class TestCutlassW4A8InactiveRoutes(CustomTestCase): if not torch.cuda.is_available(): raise unittest.SkipTest("CUDA is not available") + def test_input_scale_excludes_inactive_rows(self): + """Only local-route rows affect the scale, without changing strided inputs.""" + experts, rows, width = 2, 11, 257 + floor = 1e-10 / torch.finfo(torch.float8_e4m3fn).max + for dtype in (torch.float16, torch.bfloat16, torch.float32): + values = torch.empty((rows, width * 2), device="cuda", dtype=dtype)[:, ::2] + values.copy_( + torch.arange(rows * width, device="cuda") + .reshape(rows, width) + .remainder(53) + .sub(26) + .float() + / 8 + ) + values[9, -1] = 7.25 + ids = torch.tensor( + [ + [5, -1, -3], + [2, 0, 2], + [1, 2, -1], + [-1, -1, 2], + [2, 2, 1], + [1, 0, 2], + [2, -1, 0], + [2, 2, 2], + [0, 2, 1], + [2, 1, 2], + [-1, 2, -1], + ], + device="cuda", + dtype=torch.int64 if dtype == torch.float32 else torch.int32, + ).repeat_interleave(2, dim=1)[:, ::2] + active = ((ids >= 0) & (ids < experts)).any(dim=-1) + expected = values[active].float().abs().amax() / 448.0 + scale = torch.full((1,), floor, device="cuda", dtype=torch.float32) + + def prepare(value, route_ids, expected_scale, *, rtol=0): + source_values, source_ids = value.clone(), route_ids.clone() + scale.fill_(floor) + compute_cutlass_w4a8_moe_input_scale( + input=value, + topk_ids=route_ids, + output_s=scale, + num_local_experts=experts, + ) + torch.testing.assert_close(route_ids, source_ids, rtol=0, atol=0) + torch.testing.assert_close( + value, source_values, rtol=0, atol=0, equal_nan=True + ) + torch.testing.assert_close( + scale, + torch.as_tensor( + expected_scale, device=scale.device, dtype=scale.dtype + ).reshape_as(scale), + rtol=rtol, + atol=0, + ) + + for poison in (1e4, float("inf"), float("nan")): + with self.subTest(dtype=dtype, poison=poison): + values[~active] = poison + prepare(values, ids, expected, rtol=1e-6) + for case in ("zero", "inactive", "empty", "zero-topk", "zero-width"): + with self.subTest(dtype=dtype, case=case): + values.zero_() + if case in ("inactive", "zero-topk"): + values.fill_(float("nan")) + if case == "inactive": + ids.fill_(-1) + elif case == "zero-width": + ids.fill_(0) + ids[0, 0] = -1 + route_ids = ids + if case == "empty": + route_ids = ids[:0] + elif case == "zero-topk": + route_ids = ids[:, :0] + value = values + if case == "empty": + value = values[:0] + elif case == "zero-width": + value = values[:, :0] + prepare(value, route_ids, floor) + + def test_input_scale_and_raw_route_packing_graph_replay(self): + """Replays compact only live routes and reset the scale as routes change.""" + experts, rows, width = 2, 13, 257 + floor = 1e-10 / torch.finfo(torch.float8_e4m3fn).max + values = torch.ones((rows, width), device="cuda", dtype=torch.bfloat16) + ids = torch.zeros((rows, 2), device="cuda", dtype=torch.int32) + permutation = [None] + scale = torch.full((1,), floor, device="cuda", dtype=torch.float32) + + def run(): + scale.fill_(floor) + compute_cutlass_w4a8_moe_input_scale( + input=values, + topk_ids=ids, + output_s=scale, + num_local_experts=experts, + ) + permutation[0] = w4a8.cutlass_w4_run_moe_ep_preproess(ids) + + run() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + for case, expected in ( + ("live", 2 / 448), + ("zero", floor), + ("inactive", floor), + ("partial", 3 / 448), + ): + with self.subTest(case=case): + values.fill_(2.0 if case == "live" else 0.0) + ids.fill_(0) + if case == "inactive": + values.fill_(float("nan")) + ids.fill_(-1) + elif case == "partial": + ids.fill_(experts) + ids[0, 0], ids[0, 1] = -1, -3 + ids[-1, 1] = 1 + values.fill_(float("nan")) + values[-1].zero_() + values[-1, -1] = 3.0 + source_values, source_ids = values.clone(), ids.clone() + graph.replay() + flat_ids = source_ids.cpu().flatten() + local_indices = torch.nonzero( + (flat_ids >= 0) & (flat_ids < experts), as_tuple=True + )[0] + packed_indices = local_indices[ + torch.argsort(flat_ids[local_indices], stable=True) + ] + torch.testing.assert_close( + permutation[0].cpu()[packed_indices], + torch.arange(local_indices.numel(), dtype=torch.int32), + rtol=0, + atol=0, + ) + torch.testing.assert_close(ids, source_ids, rtol=0, atol=0) + torch.testing.assert_close( + values, source_values, rtol=0, atol=0, equal_nan=True + ) + torch.testing.assert_close( + scale, torch.full_like(scale, expected), rtol=1e-6, atol=0 + ) + def test_ep1_and_ep2_reorder_match_dense_experts(self): """Real CUDA permutation/count/finalize kernels must ignore padded routes. Only the external CUTLASS GEMMs are replaced with dense expert GEMMs; this checks routing, not native INT4 GEMM numerics. """ - experts, hidden, intermediate = 2, 128, 128 + experts, hidden, intermediate = 17, 128, 128 ids = torch.tensor( - [[1, 0], [0, -1], [-1, -1], [1, 0]], device="cuda", dtype=torch.int32 + [[1, 0], [0, -1], [experts, experts + 3], [1, -3]], + device="cuda", + dtype=torch.int32, ) weights = torch.tensor( [[0.25, 0.75], [0.6, 0.4], [0.3, 0.7], [0.8, 0.2]], device="cuda" @@ -48,19 +203,24 @@ class TestCutlassW4A8InactiveRoutes(CustomTestCase): ] dense_w2 = [eye * ((e + 1) / 2) for e in range(experts)] - def dense_reference(dynamic): + local_routes = (ids >= 0) & (ids < experts) + + def dense_reference( + dynamic, *, value=inputs, route_ids=ids, route_weights=weights + ): + local_routes = (route_ids >= 0) & (route_ids < experts) scale1 = ( - inputs[(ids >= 0).any(dim=-1)].float().abs().amax() / 448.0 + value[local_routes.any(dim=-1)].float().abs().amax() / 448.0 if dynamic else 1.0 ) activated = {} - for token in range(ids.shape[0]): - for slot in range(ids.shape[1]): - expert = int(ids[token, slot]) - if expert < 0: + for token in range(route_ids.shape[0]): + for slot in range(route_ids.shape[1]): + expert = int(route_ids[token, slot]) + if not 0 <= expert < experts: continue - quantized = (inputs[token].float() / scale1).to(torch.float8_e4m3fn) + quantized = (value[token].float() / scale1).to(torch.float8_e4m3fn) gate_up = F.linear( quantized.float() * scale1, dense_w13[expert] ).to(torch.bfloat16) @@ -72,14 +232,14 @@ class TestCutlassW4A8InactiveRoutes(CustomTestCase): if dynamic else 1.0 ) - result = torch.zeros_like(inputs, dtype=torch.float32) + result = torch.zeros_like(value, dtype=torch.float32) for (token, slot), value in activated.items(): - expert = int(ids[token, slot]) + expert = int(route_ids[token, slot]) quantized = (value / scale2).to(torch.float8_e4m3fn) down = F.linear(quantized.float() * scale2, dense_w2[expert]).to( torch.bfloat16 ) - result[token] += down.float() * weights[token, slot] + result[token] += down.float() * route_weights[token, slot] return result w13_q = torch.zeros( @@ -100,6 +260,8 @@ class TestCutlassW4A8InactiveRoutes(CustomTestCase): def dense_gemm( out, x, packed, input_scale, _weight_scale, starts, sizes, *args ): + self.assertTrue(torch.isfinite(input_scale).all().item()) + self.assertGreater(input_scale.item(), 0) out.fill_(float("nan")) matrices = dense_w13 if packed is w13_q else dense_w2 for expert in range(experts): @@ -147,14 +309,18 @@ class TestCutlassW4A8InactiveRoutes(CustomTestCase): ): value = inputs.clone() value[2].fill_(poison) - route_weights = weights.masked_fill(ids < 0, float("nan")) + route_weights = weights.masked_fill( + ~local_routes, float("nan") + ) actual = run( value, dynamic=dynamic, route_weights=route_weights ) torch.testing.assert_close( offsets, torch.tensor( - [0, 3, 5], device="cuda", dtype=torch.int32 + [0, 2] + [4] * (experts - 1), + device="cuda", + dtype=torch.int32, ), ) self.assertTrue(torch.isfinite(actual).all().item()) @@ -162,6 +328,62 @@ class TestCutlassW4A8InactiveRoutes(CustomTestCase): actual, reference.to(actual.dtype), rtol=1e-2, atol=1e-3 ) self.assertEqual(torch.count_nonzero(actual[2]).item(), 0) + torch.testing.assert_close( + value, + inputs.masked_fill( + ~local_routes.any(-1, keepdim=True), poison + ), + rtol=0, + atol=0, + equal_nan=True, + ) + if ep_size == 2: + backing = torch.tensor( + [[0, 0, 1, 0], [1, 0, 0, 0]], + device="cuda", + dtype=torch.int32, + ) + sliced_ids = backing[:, ::2] + sliced_weights = weights[:2].contiguous() + original_backing = backing.clone() + self.assertEqual(sliced_ids.stride(), (4, 2)) + self.assertTrue(sliced_weights.is_contiguous()) + for dynamic in (False, True): + with self.subTest( + ep_size=ep_size, dynamic=dynamic, sliced_ids=True + ): + actual = run( + inputs[:2], + dynamic=dynamic, + route_ids=sliced_ids, + route_weights=sliced_weights, + ) + torch.testing.assert_close( + offsets, + torch.tensor( + [0, 2] + [4] * (experts - 1), + device="cuda", + dtype=torch.int32, + ), + ) + reference = dense_reference( + dynamic, + value=inputs[:2], + route_ids=sliced_ids, + route_weights=sliced_weights, + ) + torch.testing.assert_close( + actual, reference.to(actual.dtype), rtol=1e-2, atol=1e-3 + ) + torch.testing.assert_close(backing, original_backing) + + actual = run( + torch.zeros_like(inputs), + dynamic=True, + route_weights=weights.masked_fill(~local_routes, float("nan")), + ) + self.assertTrue(torch.isfinite(actual).all().item()) + self.assertEqual(torch.count_nonzero(actual).item(), 0) inactive = torch.full_like(ids, -1) actual = run( torch.full_like(inputs, float("nan")), @@ -174,7 +396,7 @@ class TestCutlassW4A8InactiveRoutes(CustomTestCase): torch.testing.assert_close( ids, torch.tensor( - [[1, 0], [0, -1], [-1, -1], [1, 0]], + [[1, 0], [0, -1], [experts, experts + 3], [1, -3]], device="cuda", dtype=torch.int32, ), diff --git a/test/registered/moe/test_flashinfer_fp8_routed.py b/test/registered/moe/test_flashinfer_fp8_routed.py new file mode 100644 index 0000000000..b074fbb802 --- /dev/null +++ b/test/registered/moe/test_flashinfer_fp8_routed.py @@ -0,0 +1,294 @@ +"""Native FP8 routing keeps live-count bounds, score values, and deferred top-k.""" + +import unittest +from contextlib import ExitStack, nullcontext +from types import SimpleNamespace + +import torch + +from sglang.srt.configs.shared_experts import SharedExpertsConfig +from sglang.srt.layers.quantization.fp8 import Fp8Config +from sglang.srt.runtime_context import ( + get_context, + get_flags, + get_model, + get_parallel, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +# This uses one GPU; the pool supplies the required Blackwell FP8 kernels. +register_cuda_ci(est_time=20, stage="extra-b", runner_config="4-gpu-b200") + + +@unittest.skipUnless( + torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 10, + "Requires Blackwell FlashInfer TRTLLM FP8 kernels", +) +class TestFlashinferFp8Routed(CustomTestCase): + def setUp(self): + super().setUp() + from sglang.srt.layers.moe.utils import ( + MoeA2ABackend, + MoeRunnerBackend, + initialize_moe_config, + ) + + scopes = ExitStack() + self.addCleanup(scopes.close) + scopes.enter_context( + get_context().override_server_args( + model_path="dummy", device="cuda", enable_eplb=False + ) + ) + initialize_moe_config() + scopes.enter_context( + get_flags().moe.override( + runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM, + a2a_backend=MoeA2ABackend.NONE, + ) + ) + scopes.enter_context( + get_parallel().override( + tp_size=1, + tp_rank=0, + tp_group=None, + attn_tp_size=1, + attn_tp_rank=0, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + ) + ) + scopes.enter_context(get_model().override(shared_experts=SharedExpertsConfig())) + scopes.enter_context(torch.no_grad()) + + @staticmethod + def _group_roundtrip(value): + grouped = value.float().reshape(value.shape[0], -1, 128) + scales = grouped.abs().amax(-1, keepdim=True).clamp_min(1e-10) / 448 + return ((grouped / scales).to(torch.float8_e4m3fn).float() * scales).reshape_as( + value + ) + + def _check_live_routes(self, *, block_quant, dtype): + from sglang.srt.layers.moe.moe_runner import flashinfer_trtllm as runner + from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig + from sglang.srt.layers.moe.token_dispatcher.standard import ( + StandardDispatchOutput, + ) + from sglang.srt.layers.moe.topk import TopK, TopKOutputChecker + from sglang.srt.layers.moe.utils import MoeRunnerBackend, RoutingMethodType + from sglang.srt.model_executor.forward_context import ( + ForwardContext, + forward_context, + ) + from sglang.srt.models.llama4 import Llama4MoE + + m, h, inter, experts = 4, 256, 256, 4 + clean = ( + ((torch.arange(m * h, device="cuda").reshape(m, h) % 17) + 64) / 128 + ).to(dtype) + hidden = clean.clone() + clean_logits = torch.full((m, experts), -2.0, device="cuda") + selected = torch.arange(m, device="cuda") % experts + clean_logits[torch.arange(m, device="cuda"), selected] = ( + 1.0 + torch.arange(m, device="cuda") / 8 + ) + logits = clean_logits.clone() + live = torch.tensor(m, device="cuda", dtype=torch.int32) + w13 = torch.zeros((experts, 2 * inter, h), device="cuda") + w2 = torch.zeros((experts, h, inter), device="cuda") + diagonal = torch.arange(h, device="cuda") + for expert in range(experts): + w13[expert, diagonal, diagonal] = 0.5 + expert / 8 + w13[expert, inter + diagonal, diagonal] = 0.25 + w2[expert, diagonal, diagonal] = 0.5 + w13q, w2q = w13.to(torch.float8_e4m3fn), w2.to(torch.float8_e4m3fn) + a1, a2 = ( + torch.tensor([1 / 128], device="cuda"), + torch.tensor([1 / 256], device="cuda"), + ) + quant_config = Fp8Config( + is_checkpoint_fp8_serialized=True, + weight_block_size=[128, 128] if block_quant else None, + ) + quant_args = dict( + w13_weight=w13q, + w2_weight=w2q, + global_num_experts=experts, + local_expert_offset=0, + local_num_experts=experts, + intermediate_size=inter, + routing_method_type=RoutingMethodType.Llama4, + block_quant=block_quant, + ) + if block_quant: + # Fp8MoEMethod's TRTLLM loader stores the up half before the gate. + quant_args.update( + w13_weight=torch.cat((w13[:, inter:], w13[:, :inter]), dim=1).to( + torch.float8_e4m3fn + ), + weight_block_k=quant_config.weight_block_size[1], + w13_weight_scale_inv=torch.ones( + (experts, 2 * inter // 128, h // 128), device="cuda" + ), + w2_weight_scale_inv=torch.ones( + (experts, h // 128, inter // 128), device="cuda" + ), + ) + else: + layer = torch.nn.Module() + layer.moe_runner_config = SimpleNamespace(is_gated=True) + for name, value in { + "w13_weight": w13q, + "w2_weight": w2q, + "w13_input_scale": a1, + "w2_input_scale": a2, + "w13_weight_scale": torch.ones(experts, device="cuda"), + "w2_weight_scale": torch.ones(experts, device="cuda"), + }.items(): + setattr(layer, name, torch.nn.Parameter(value, requires_grad=False)) + # Follow ModelOptFp8MoEMethod's preparation from logical gate/up. + runner.align_fp8_moe_weights_for_flashinfer_trtllm( + layer, swap_w13_halves=True + ) + quant_args.update( + w13_weight=layer.w13_weight, + w2_weight=layer.w2_weight, + w13_input_scale=layer.w13_input_scale, + output1_scales_scalar=layer.output1_scales_scalar, + output1_scales_gate_scalar=layer.output1_scales_gate_scalar, + output2_scales_scalar=layer.output2_scales_scalar, + ) + quant = runner.FlashInferTrtllmFp8MoeQuantInfo(**quant_args) + # Llama4's TopK owns k=1; its expert runner need not duplicate that setting. + config = MoeRunnerConfig(activation="silu") + router = TopK( + top_k=1, + layer_id=0, + renormalize=False, + custom_routing_function=Llama4MoE.custom_routing_function, + ) + router.bind_runner_backend( + MoeRunnerBackend.FLASHINFER_TRTLLM, + routing_method_type=RoutingMethodType.Llama4, + ) + context = ForwardContext( + attn_backend=object(), moe_num_token_non_padded=lambda: live + ) + without_bound = ForwardContext(attn_backend=object()) + + def entry(deferred, *, bounded=True): + with forward_context(context if bounded else without_bound): + lazy = router.forward_cuda(hidden, logits) + self.assertTrue(TopKOutputChecker.format_is_bypassed(lazy)) + self.assertIs(lazy.num_token_non_padded, live if bounded else None) + output = runner.fused_experts_none_to_flashinfer_trtllm_fp8( + StandardDispatchOutput(hidden, None, lazy), quant, config + ).hidden_states + if deferred: + self.assertEqual(output.top_k, 1) + return runner.finalize_flashinfer_trtllm_deferred_output(output, None) + return output + + def reference(real): + expected = torch.zeros((m, h), device="cuda", dtype=torch.bfloat16) + if real == 0: + return expected + x = clean[:real].float() + dequant = ( + self._group_roundtrip(x) + if block_quant + else (x / a1).to(torch.float8_e4m3fn).float() * a1 + ) + scores = clean_logits[:real].max(-1).values.sigmoid().to(dtype).float() + if not block_quant: + # Llama4 supplies BF16 token scales to the native FP8 GEMM1. + dequant = dequant * scores.to(torch.bfloat16).float()[:, None] + # Match this native runner's final-weight contract, independently + # of its route materialization and grouped GEMM layout. + for token in range(real): + expert = int(selected[token]) + gate, up = torch.nn.functional.linear( + dequant[token], w13[expert] + ).chunk(2) + act = torch.nn.functional.silu(gate) * up + aq = ( + self._group_roundtrip(act[None])[0] + if block_quant + else (act / a2).to(torch.float8_e4m3fn).float() * a2 + ) + expected[token] = ( + torch.nn.functional.linear(aq, w2[expert]) * scores[token] + ) + return expected + + for deferred in (False, True) if block_quant else (False,): + with ( + runner.flashinfer_trtllm_deferred_finalize_context() + if deferred + else nullcontext() + ): + valid_logits = entry(deferred, bounded=False).clone() + valid_routed = entry(deferred).clone() + torch.testing.assert_close( + valid_routed, valid_logits, rtol=0.01, atol=0.001 + ) + for _ in range(3): + entry(deferred) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = entry(deferred) + for state, real in ( + ("valid", m), + ("partial", 2), + ("inactive", 0), + ("recovery", m), + ): + with self.subTest(deferred=deferred, state=state): + live.fill_(real) + hidden.copy_(clean) + logits.copy_(clean_logits) + hidden[real:] = logits[real:] = float("nan") + saved_hidden, saved_logits = hidden.clone(), logits.clone() + eager = entry(deferred).clone() + graph.replay() + torch.cuda.synchronize() + torch.testing.assert_close( + eager.float(), + reference(real).float(), + rtol=0.01, + atol=0.001, + ) + torch.testing.assert_close( + captured, eager, rtol=0.01, atol=0.001 + ) + self.assertTrue(bool(torch.isfinite(captured).all())) + self.assertTrue(bool((captured[real:] == 0).all())) + torch.testing.assert_close( + hidden, saved_hidden, rtol=0, atol=0, equal_nan=True + ) + torch.testing.assert_close( + logits, saved_logits, rtol=0, atol=0, equal_nan=True + ) + del graph, captured + live.fill_(m) + hidden.copy_(clean) + logits.copy_(clean_logits) + + def test_block_fp8_live_routes_and_deferred_finalize(self): + self._check_live_routes(block_quant=True, dtype=torch.bfloat16) + + def test_per_tensor_fp8_half_routes(self): + self._check_live_routes(block_quant=False, dtype=torch.float16) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/kernels/test_kernels_namespace.py b/test/registered/unit/kernels/test_kernels_namespace.py index 9c5f0d5d33..2f04acfc7b 100644 --- a/test/registered/unit/kernels/test_kernels_namespace.py +++ b/test/registered/unit/kernels/test_kernels_namespace.py @@ -32,6 +32,21 @@ def test_single_backend_resolves_without_backend(): ) +def test_shuffle_weighted_sum_has_one_native_primary_backend(): + specs = K.registry.get("moe.shuffle_weighted_sum") + assert len(specs) == 1 + spec = K.select_kernel("moe.shuffle_weighted_sum") + assert spec is specs[0] and spec.backend is KernelBackend.JIT + assert ( + spec.target + == "sglang.kernels.ops.moe.shuffle_weighted_sum:shuffle_weighted_sum" + ) + assert K.capabilities_satisfied(spec.capabilities, _SM90) + assert not K.capabilities_satisfied(spec.capabilities, _CPU) + with pytest.raises(KeyError): + K.select_kernel("moe.shuffle_weighted_sum", backend=KernelBackend.TRITON) + + def test_fp8_scaled_mm_requires_explicit_registry_backend(monkeypatch): monkeypatch.setattr(sel, "_platform", lambda: _SM90) with pytest.raises(ValueError, match="multiple backends"): diff --git a/test/registered/unit/layers/moe/test_standard_dispatcher.py b/test/registered/unit/layers/moe/test_standard_dispatcher.py index 289b023c15..811d072bf3 100644 --- a/test/registered/unit/layers/moe/test_standard_dispatcher.py +++ b/test/registered/unit/layers/moe/test_standard_dispatcher.py @@ -100,8 +100,12 @@ class TestLayerRunnerDispatch(CustomTestCase): top_k=2, ) - def _build_layer(self, method, *, routing_method_type=None): - with patch.object(method, "create_weights"): + def _build_layer(self, method, *, scheme=None, routing_method_type=None): + def create_weights(*, layer, **kwargs): + if scheme is not None: + layer.scheme = scheme + + with patch.object(method, "create_weights", side_effect=create_weights): return FusedMoE( num_experts=4, top_k=2, @@ -371,6 +375,9 @@ class TestLayerRunnerDispatch(CustomTestCase): def test_direct_kernels_declare_their_execution_contract(self): """A missing or dormant runner must not change a direct kernel's ID ABI.""" + from sglang.srt.layers.quantization.compressed_tensors.compressed_tensors import ( + CompressedTensorsFusedMoEMethod, + ) from sglang.srt.layers.quantization.compressed_tensors.schemes.compressed_tensors_w4a4_mxint4_moe import ( CompressedTensorsMxInt4MoE, ) @@ -380,18 +387,30 @@ class TestLayerRunnerDispatch(CustomTestCase): ) cases = [ - (CompressedTensorsMxInt4MoE, MoeRunnerBackend.FLASHINFER_TRTLLM, False), - (Mxfp4FlashinferTrtllmMoEMethod, MoeRunnerBackend.FLASHINFER_MXFP4, True), - (ModelOptFp8MoEMethod, MoeRunnerBackend.FLASHINFER_TRTLLM, False), + ( + CompressedTensorsFusedMoEMethod, + CompressedTensorsMxInt4MoE, + MoeRunnerBackend.FLASHINFER_TRTLLM, + False, + ), + ( + Mxfp4FlashinferTrtllmMoEMethod, + None, + MoeRunnerBackend.FLASHINFER_MXFP4, + True, + ), + (ModelOptFp8MoEMethod, None, MoeRunnerBackend.FLASHINFER_TRTLLM, False), ] - for method_type, backend, fp4 in cases: + for method_type, scheme_type, backend, fp4 in cases: with ( self.subTest(method=method_type.__name__), get_flags().moe.override(runner_backend=backend), ): method = object.__new__(method_type) layer = self._build_layer( - method, routing_method_type=RoutingMethodType.DeepSeekV3 + method, + scheme=object.__new__(scheme_type) if scheme_type else None, + routing_method_type=RoutingMethodType.DeepSeekV3, ) self.assertEqual(layer.runner_backend, backend) self.assertTrue(layer.dispatcher.skip_local_expert_mapping) @@ -427,21 +446,10 @@ class TestLayerRunnerDispatch(CustomTestCase): CompressedTensorsMxInt4MoE, ) - scheme = object.__new__(CompressedTensorsMxInt4MoE) - method = object.__new__(CompressedTensorsFusedMoEMethod) - - def create_weights(*, layer, **kwargs): - layer.scheme = scheme - - with patch.object(method, "create_weights", side_effect=create_weights): - layer = FusedMoE( - num_experts=4, - top_k=2, - hidden_size=4, - intermediate_size=96, - layer_id=0, - quant_method=method, - ) + layer = self._build_layer( + object.__new__(CompressedTensorsFusedMoEMethod), + scheme=object.__new__(CompressedTensorsMxInt4MoE), + ) self.assertEqual(layer.runner_backend, MoeRunnerBackend.FLASHINFER_TRTLLM) self.assertTrue(layer.use_flashinfer_trtllm_moe) self.assertTrue(layer.dispatcher.skip_local_expert_mapping) @@ -453,6 +461,7 @@ class TestLayerRunnerDispatch(CustomTestCase): from sglang.srt.layers.moe.fused_moe_triton.aiter_mxfp4_w4a8_moe import ( build_ep_expert_map, ) + from sglang.srt.layers.quantization.quark.quark import QuarkFusedMoEMethod from sglang.srt.layers.quantization.quark.schemes.quark_w4a8_mxfp4_moe import ( QuarkW4A8MXFp4MoE, ) @@ -462,7 +471,10 @@ class TestLayerRunnerDispatch(CustomTestCase): "_use_gfx1250_w4a8", True, ): - expert_parallel = self._build_layer(object.__new__(QuarkW4A8MXFp4MoE)) + expert_parallel = self._build_layer( + object.__new__(QuarkFusedMoEMethod), + scheme=object.__new__(QuarkW4A8MXFp4MoE), + ) with get_parallel().override( tp_size=1, tp_rank=0, @@ -471,7 +483,10 @@ class TestLayerRunnerDispatch(CustomTestCase): moe_ep_size=1, moe_ep_rank=0, ): - single_rank = self._build_layer(object.__new__(QuarkW4A8MXFp4MoE)) + single_rank = self._build_layer( + object.__new__(QuarkFusedMoEMethod), + scheme=object.__new__(QuarkW4A8MXFp4MoE), + ) for layer in (expert_parallel, single_rank): self.assertIsNone(layer.runner_backend) self.assertFalse(layer.dispatcher.skip_local_expert_mapping) diff --git a/test/registered/unit/layers/moe/test_standard_dispatcher_runner_backend.py b/test/registered/unit/layers/moe/test_standard_dispatcher_runner_backend.py index ee5639bb9c..c5cfd3d356 100644 --- a/test/registered/unit/layers/moe/test_standard_dispatcher_runner_backend.py +++ b/test/registered/unit/layers/moe/test_standard_dispatcher_runner_backend.py @@ -17,7 +17,11 @@ import torch from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig from sglang.srt.layers.moe.token_dispatcher import standard from sglang.srt.layers.moe.topk import StandardTopKOutput -from sglang.srt.layers.moe.utils import MoeA2ABackend, MoeRunnerBackend +from sglang.srt.layers.moe.utils import ( + MoeA2ABackend, + MoeRunnerBackend, + RoutingMethodType, +) from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.test.ci.ci_register import register_cpu_ci @@ -152,6 +156,120 @@ def test_fp8_triton_fallback_reaches_the_dispatcher(cutlass_pin): assert not layer.dispatcher.skip_local_expert_mapping +@pytest.mark.parametrize( + "backend, requested, expected", + [ + (MoeRunnerBackend.FLASHINFER_TRTLLM, None, RoutingMethodType.DeepSeekV3), + (MoeRunnerBackend.FLASHINFER_TRTLLM_ROUTED, None, RoutingMethodType.DeepSeekV3), + ( + MoeRunnerBackend.FLASHINFER_TRTLLM, + RoutingMethodType.Default, + RoutingMethodType.Default, + ), + (MoeRunnerBackend.TRITON, None, None), + (MoeRunnerBackend.FLASHINFER_CUTLASS, None, None), + ], +) +def test_fp8_publishes_only_the_selected_runners_policy( + ep2_rank0, backend, requested, expected +): + """The layer, router binding and consumer must share the actual FP8 policy.""" + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE + from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod + + ep2_rank0.runner_backend = backend + method = Fp8MoEMethod( + Fp8Config(is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128]) + ) + with patch.object(method, "create_weights"): + layer = FusedMoE( + num_experts=NUM_EXPERTS, + top_k=2, + hidden_size=16, + intermediate_size=128, + layer_id=0, + quant_method=method, + routing_method_type=requested, + ) + assert layer.routing_method_type == expected + assert layer.moe_runner_config.routing_method_type == expected + assert layer.runner_backend == method.get_moe_runner_backend(layer) + assert layer.dispatcher.skip_local_expert_mapping == ( + layer.runner_backend.is_flashinfer_trtllm() + or layer.runner_backend.is_flashinfer_trtllm_routed() + ) + + +def test_modelopt_undeclared_policy_keeps_triton_layout_and_local_ids(ep2_rank0): + """Publishing another method's default must not enable ModelOpt native repacking.""" + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE + from sglang.srt.layers.quantization import modelopt_quant + + ep2_rank0.runner_backend = MoeRunnerBackend.FLASHINFER_TRTLLM + with patch.object(modelopt_quant, "cutlass_fp8_supported", return_value=False): + method = modelopt_quant.ModelOptFp8MoEMethod( + modelopt_quant.ModelOptFp8Config(is_checkpoint_fp8_serialized=True) + ) + with patch.object(method, "create_weights"): + layer = FusedMoE( + num_experts=NUM_EXPERTS, + top_k=2, + hidden_size=16, + intermediate_size=128, + layer_id=0, + quant_method=method, + ) + assert layer.routing_method_type is None + assert layer.moe_runner_config.routing_method_type is None + assert layer.runner_backend.is_triton() + assert not layer.dispatcher.skip_local_expert_mapping + ids = torch.tensor([[0, 5], [3, 7]], dtype=torch.int32) + assert _dispatch_ids(layer.dispatcher, ids).tolist() == [[0, -1], [3, -1]] + + +@pytest.mark.parametrize( + "backend, requested, expected", + [ + (MoeRunnerBackend.FLASHINFER_TRTLLM, None, RoutingMethodType.DeepSeekV3), + ( + MoeRunnerBackend.FLASHINFER_TRTLLM, + RoutingMethodType.Default, + RoutingMethodType.Default, + ), + (MoeRunnerBackend.TRITON, None, None), + ], +) +def test_hybrid_method_preserves_its_gpu_policy( + ep2_rank0, backend, requested, expected +): + """The base query must not shadow the wrapped GPU method's routing contract.""" + from sglang.srt.layers.moe import kt_ep_wrapper + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE + from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod + + ep2_rank0.runner_backend = backend + gpu = Fp8MoEMethod( + Fp8Config(is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128]) + ) + with patch.object(kt_ep_wrapper, "KTRANSFORMERS_AVAILABLE", True): + method = kt_ep_wrapper.KTEPWrapperMethod( + gpu, SimpleNamespace(num_gpu_experts=4) + ) + with patch.object(method, "create_weights"): + layer = FusedMoE( + num_experts=NUM_EXPERTS, + top_k=2, + hidden_size=16, + intermediate_size=128, + layer_id=0, + quant_method=method, + routing_method_type=requested, + ) + assert layer.routing_method_type == expected + assert layer.moe_runner_config.routing_method_type == expected + assert layer.runner_backend == gpu.runner.runner_backend + + if __name__ == "__main__": import sys diff --git a/test/registered/unit/layers/moe/test_topk_routing_inputs.py b/test/registered/unit/layers/moe/test_topk_routing_inputs.py index a051776a25..9ca525b050 100644 --- a/test/registered/unit/layers/moe/test_topk_routing_inputs.py +++ b/test/registered/unit/layers/moe/test_topk_routing_inputs.py @@ -1,7 +1,10 @@ """Routing follows the owning layer and active batch across target/draft and TBO.""" +import inspect +import sys import unittest -from types import SimpleNamespace +from enum import IntEnum +from types import ModuleType, SimpleNamespace from unittest.mock import patch import torch @@ -402,6 +405,371 @@ class TestTopKRoutingInputs(CustomTestCase): piecewise = FusedMoE.forward(layer, self.hidden, output) torch.testing.assert_close(piecewise.topk_ids, materialized.topk_ids) + def test_piecewise_lazy_routing_keeps_a_bound_without_placement(self): + """A logits-only opaque op must not drop a row bound on graph entry.""" + router = self._router(output_format=TopKOutputFormat.BYPASSED) + with get_exec().moe.override(ep_dispatch_algorithm=None): + output = router.forward_cuda( + self.hidden, self.logits, num_token_non_padded=torch.tensor(2) + ) + self.assertIsNone(output.expert_location_dispatch_info) + layer = SimpleNamespace( + _use_ascend_fuseep=False, + forward_impl=lambda hidden, routes, **kwargs: routes.to_standard(), + ) + with patch( + "sglang.srt.layers.moe.fused_moe_triton.layer.is_in_tc_piecewise_cuda_graph", + return_value=True, + ): + materialized = FusedMoE.forward(layer, self.hidden, output) + self.assertEqual( + materialized.topk_ids.tolist(), [[0, 1], [0, 1], [-1, -1], [-1, -1]] + ) + + def test_fp8_consumers_keep_lazy_padding_and_placement(self): + """Both FP8 formats consume physical IDs and ignore poisoned inactive scores.""" + from sglang.srt.layers.moe.moe_runner import flashinfer_trtllm + from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig + from sglang.srt.layers.moe.token_dispatcher.standard import ( + StandardDispatchOutput, + ) + from sglang.srt.layers.moe.utils import RoutingMethodType + + provider = ModuleType("flashinfer.fused_moe") + provider.Fp8QuantizationType = IntEnum( + "Fp8QuantizationType", {"DeepSeekFp8": 0, "MxFp8": 1} + ) + + def score_router(**kwargs): + weights, ids = fused_topk_torch_native(**kwargs) + return weights.to(score_dtype), ids + + def routed_kernel(*, topk_ids, output, routing_method_type, **kwargs): + ids, weights = topk_ids + self.assertEqual(routing_method_type, RoutingMethodType.TopK) + self.assertEqual(kwargs["top_k"], ids.shape[1]) + self.assertEqual( + weights.dtype, + torch.float32 if score_dtype == torch.float16 else score_dtype, + ) + self.assertTrue(ids.is_contiguous() and weights.is_contiguous()) + self.assertTrue(bool((ids[real:] == -1).all())) + self.assertEqual(ids[:real].tolist(), [expected_ids] * real) + values = (ids.float() + 1) * torch.where(ids >= 0, weights, 0) + output.copy_(values.sum(-1, keepdim=True).expand_as(output)) + + provider.trtllm_fp8_block_scale_routed_moe = routed_kernel + provider.trtllm_fp8_per_tensor_scale_routed_moe = routed_kernel + + def logits_kernel(*, routing_logits, output=None, **kwargs): + weights, ids = routing_logits.float().softmax(-1).topk(2) + weights = weights / weights.sum(-1, keepdim=True) * 2.5 + values = ((ids.float() + 1) * weights).sum(-1, keepdim=True) + result = values.expand_as(self.hidden) + if output is None: + return result.to(torch.bfloat16) + output.copy_(result) + return output + + provider.trtllm_fp8_block_scale_moe = logits_kernel + provider.trtllm_fp8_per_tensor_scale_moe = logits_kernel + for block_quant in (True, False): + quant_info = flashinfer_trtllm.FlashInferTrtllmFp8MoeQuantInfo( + w13_weight=torch.empty((4, 4, 2)), + w2_weight=torch.empty((4, 2, 2)), + global_num_experts=4, + local_expert_offset=0, + local_num_experts=4, + intermediate_size=2, + routing_method_type=RoutingMethodType.RenormalizeNaive, + block_quant=block_quant, + weight_block_k=128, + w13_weight_scale_inv=torch.ones((4, 1, 1)), + w2_weight_scale_inv=torch.ones((4, 1, 1)), + w13_input_scale=torch.ones(1), + output1_scales_scalar=torch.ones(4), + output1_scales_gate_scalar=torch.ones(4), + output2_scales_scalar=torch.ones(4), + ) + for score_dtype in (torch.float16, torch.bfloat16, torch.float32): + for placement in (False, True): + expected_ids = [2, 0] if placement else [0, 1] + for real in (3, 1, 0, 4): + with self.subTest( + block_quant=block_quant, + score_dtype=score_dtype, + placement=placement, + real=real, + ): + hidden = self.hidden.clone() + hidden[real:] = float("nan") + logits = self.logits.clone() + logits[real:] = float("nan") + router = self._router( + output_format=TopKOutputFormat.BYPASSED, + routed_scaling_factor=2.5, + apply_routed_scaling_factor_on_output=True, + ) + router.topk_config.custom_routing_function = score_router + with get_exec().moe.override( + ep_dispatch_algorithm="static" if placement else None + ): + lazy = router.forward_cuda( + hidden, + logits, + num_token_non_padded=torch.tensor(real), + ) + with ( + get_parallel().override(tp_group=None), + patch.dict( + sys.modules, {"flashinfer.fused_moe": provider} + ), + patch.object( + flashinfer_trtllm, + "trtllm_fp8_block_scale_moe_out_wrapper", + inspect.unwrap( + flashinfer_trtllm.trtllm_fp8_block_scale_moe_out_wrapper + ), + ), + patch.object( + flashinfer_trtllm, + "trtllm_fp8_per_tensor_scale_moe_wrapper", + inspect.unwrap( + flashinfer_trtllm.trtllm_fp8_per_tensor_scale_moe_wrapper + ), + ), + patch.object( + flashinfer_trtllm, + "trtllm_fp8_block_scale_routed_moe_out_wrapper", + inspect.unwrap( + flashinfer_trtllm.trtllm_fp8_block_scale_routed_moe_out_wrapper + ), + ), + patch.object( + flashinfer_trtllm, + "trtllm_fp8_per_tensor_scale_routed_moe_out_wrapper", + inspect.unwrap( + flashinfer_trtllm.trtllm_fp8_per_tensor_scale_routed_moe_out_wrapper + ), + ), + patch.object( + flashinfer_trtllm, + "per_token_group_quant_fp8", + return_value=(hidden, torch.ones((1, 1))), + ), + patch.object( + flashinfer_trtllm, + "scaled_fp8_quant", + return_value=(hidden, torch.ones(1)), + ), + ): + result = flashinfer_trtllm.fused_experts_none_to_flashinfer_trtllm_fp8( + StandardDispatchOutput(hidden, None, lazy), + quant_info, + MoeRunnerConfig( + activation="silu", + routed_scaling_factor=2.5, + ), + ).hidden_states + scores = self.logits[:, :2].softmax(-1).to(score_dtype) + scores *= 2.5 + expected = ( + (scores * (torch.tensor(expected_ids).float() + 1)) + .sum(-1, keepdim=True) + .expand_as(result) + .clone() + ) + expected[real:] = 0 + torch.testing.assert_close( + result.float(), expected, rtol=0.005, atol=0.005 + ) + self.assertTrue(bool(torch.isfinite(result).all())) + + def test_fp8_route_scores_preserve_values_and_supported_storage(self): + """FP16 external scores widen exactly; native BF16/FP32 scores keep storage.""" + from torch.utils._python_dispatch import TorchDispatchMode + + from sglang.srt.layers.moe.moe_runner import flashinfer_trtllm + from sglang.srt.layers.moe.topk import StandardTopKOutput + + class ObserveScoreCopies(TorchDispatchMode): + def __init__(self): + self.copies = [] + + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + if func in ( + torch.ops.aten._to_copy.default, + torch.ops.aten.clone.default, + ): + self.copies.append(func) + return func(*args, **(kwargs or {})) + + for dtype in (torch.float16, torch.bfloat16, torch.float32): + for strided in (False, True): + with self.subTest(dtype=dtype, strided=strided): + # Values expose FP16-to-BF16 double rounding and preserve the + # payload of dropped routes for the consumer to ignore. + storage = torch.tensor( + [ + [0.333251953125, 9.0, 2.0**-20, 9.0], + [float("nan"), 9.0, float("inf"), 9.0], + ], + dtype=dtype, + ) + weights = ( + storage[:, ::2] if strided else storage[:, ::2].contiguous() + ) + ids = torch.tensor([[0, 1], [-1, -1]], dtype=torch.int32) + original = weights.clone() + with ObserveScoreCopies() as observed: + routing = flashinfer_trtllm._get_routing_for_flashinfer_routed( + StandardTopKOutput(weights, ids, self.logits[:2]), + promote_half_scores=True, + ) + if dtype == torch.float16: + self.assertEqual( + observed.copies, [torch.ops.aten._to_copy.default] + ) + else: + self.assertEqual(len(observed.copies), int(strided)) + actual_ids, actual_weights = routing + self.assertIs(actual_ids, ids) + expected_dtype = torch.float32 if dtype == torch.float16 else dtype + self.assertEqual(actual_weights.dtype, expected_dtype) + self.assertTrue(actual_weights.is_contiguous()) + torch.testing.assert_close( + actual_weights.float(), + original.float(), + rtol=0, + atol=0, + equal_nan=True, + ) + torch.testing.assert_close( + weights, original, rtol=0, atol=0, equal_nan=True + ) + if dtype != torch.float16 and not strided: + self.assertIs(actual_weights, weights) + + def test_fp8_deferred_finalize_keeps_materialized_routes(self): + """Padding and placement retain the deferred GEMM2/score/permutation ABI.""" + from sglang.srt.layers.moe.moe_runner import flashinfer_trtllm + from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig + from sglang.srt.layers.moe.token_dispatcher.standard import ( + StandardDispatchOutput, + ) + from sglang.srt.layers.moe.utils import RoutingMethodType + + provider = ModuleType("flashinfer.fused_moe") + provider.Fp8QuantizationType = IntEnum( + "Fp8QuantizationType", {"DeepSeekFp8": 0, "MxFp8": 1} + ) + returned = [] + + def deferred_result(ids, weights, kwargs): + self.assertFalse(kwargs["do_finalize"]) + self.assertNotIn("output", kwargs) + self.assertEqual(kwargs["top_k"], ids.shape[1]) + self.assertEqual(weights.dtype, torch.float32) + indices = torch.arange(ids.numel(), dtype=torch.int32).reshape_as(ids) + indices[ids < 0] = -1 + gemm2 = (ids.clamp_min(0).reshape(-1, 1) + 1).to(torch.bfloat16) + gemm2 = gemm2.expand(-1, self.hidden.shape[1]).contiguous() + returned[:] = [gemm2, weights, indices] + return returned + + def routed_kernel(*, topk_ids, **kwargs): + ids, weights = topk_ids + expected_ids = [2, 0] if placement else [0, 1] + self.assertEqual(ids[:real].tolist(), [expected_ids] * real) + self.assertTrue(bool((ids[real:] == -1).all())) + self.assertEqual(kwargs["routing_method_type"], RoutingMethodType.TopK) + return deferred_result(ids, weights, kwargs) + + def logits_kernel(*, routing_logits, **kwargs): + self.assertIs(routing_logits, logits) + self.assertEqual( + kwargs["routing_method_type"], RoutingMethodType.RenormalizeNaive + ) + weights, ids = routing_logits.softmax(-1).topk(2) + weights = weights / weights.sum(-1, keepdim=True) * 2.5 + return deferred_result(ids, weights, kwargs) + + provider.trtllm_fp8_block_scale_routed_moe = routed_kernel + provider.trtllm_fp8_block_scale_moe = logits_kernel + quant_info = flashinfer_trtllm.FlashInferTrtllmFp8MoeQuantInfo( + w13_weight=torch.empty((4, 4, 2)), + w2_weight=torch.empty((4, 2, 2)), + global_num_experts=4, + local_expert_offset=0, + local_num_experts=4, + intermediate_size=2, + routing_method_type=RoutingMethodType.RenormalizeNaive, + block_quant=True, + weight_block_k=128, + w13_weight_scale_inv=torch.ones((4, 1, 1)), + w2_weight_scale_inv=torch.ones((4, 1, 1)), + ) + for bound, placement in ((None, False), (2, False), (2, True), (0, True)): + real = 4 if bound is None else bound + with self.subTest(bound=bound, placement=placement): + hidden, logits = self.hidden.clone(), self.logits.clone() + hidden[real:] = logits[real:] = float("nan") + router = self._router( + output_format=TopKOutputFormat.BYPASSED, + routed_scaling_factor=2.5, + apply_routed_scaling_factor_on_output=True, + ) + with get_exec().moe.override( + ep_dispatch_algorithm="static" if placement else None + ): + lazy = router.forward_cuda( + hidden, + logits, + num_token_non_padded=( + None if bound is None else torch.tensor(bound) + ), + ) + with ( + patch.dict(sys.modules, {"flashinfer.fused_moe": provider}), + patch.object( + flashinfer_trtllm, + "per_token_group_quant_fp8", + return_value=(hidden, torch.ones((1, 1))), + ), + patch.object( + flashinfer_trtllm, "use_symmetric_memory" + ) as allocation, + flashinfer_trtllm.flashinfer_trtllm_deferred_finalize_context(), + ): + result = ( + flashinfer_trtllm.fused_experts_none_to_flashinfer_trtllm_fp8( + StandardDispatchOutput(hidden, None, lazy), + quant_info, + MoeRunnerConfig( + activation="silu", + routed_scaling_factor=2.5, + ), + ).hidden_states + ) + allocation.assert_not_called() + self.assertIsInstance( + result, flashinfer_trtllm.FlashInferTrtllmDeferredFinalizeOutput + ) + self.assertIs(result.gemm2_out, returned[0]) + self.assertIs(result.expert_weights, returned[1]) + self.assertIs(result.expanded_idx_to_permuted_idx, returned[2]) + self.assertEqual(result.top_k, 2) + self.assertTrue(bool((returned[2][real:] == -1).all())) + self.assertEqual(result.expert_weights.dtype, torch.float32) + self.assertTrue( + bool(torch.isfinite(result.expert_weights[:real]).all()) + ) + # Dropped permutation entries exclude these still-poisoned scores. + self.assertTrue(bool(torch.isnan(result.expert_weights[real:]).all())) + torch.testing.assert_close( + result.expert_weights[:real].sum(-1), torch.full((real,), 2.5) + ) + def test_in_range_draft_loads_experts_in_its_routing_order(self): """Draft expert weights stay in logical order despite a permuted target map.""" outputs = [] diff --git a/test/registered/unit/layers/quantization/test_fp8_moe_runner_ownership.py b/test/registered/unit/layers/quantization/test_fp8_moe_runner_ownership.py index 80fa2c81fc..b28441240b 100644 --- a/test/registered/unit/layers/quantization/test_fp8_moe_runner_ownership.py +++ b/test/registered/unit/layers/quantization/test_fp8_moe_runner_ownership.py @@ -11,17 +11,33 @@ import math import sys import types import unittest +from contextlib import ExitStack from types import SimpleNamespace from unittest.mock import patch import torch +from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.moe_runner.aiter import AiterQuantType from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig -from sglang.srt.layers.moe.utils import MoeRunnerBackend +from sglang.srt.layers.moe.utils import ( + MoeA2ABackend, + MoeRunnerBackend, + RoutingMethodType, +) from sglang.srt.layers.quantization import fp8 as fp8_module +from sglang.srt.layers.quantization import mxfp4_flashinfer_cutlass_moe as cutlass_mxfp4 from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod -from sglang.srt.runtime_context import get_flags +from sglang.srt.layers.quantization.modelopt_quant import ( + HybridFp8NvFp4Config, + ModelOptFp4Config, +) +from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( + Mxfp4FlashinferTrtllmMoEMethod, +) +from sglang.srt.layers.quantization.mxfp4_humming_moe import Mxfp4HummingMoEMethod +from sglang.srt.layers.quantization.mxfp4_marlin_moe import Mxfp4MarlinMoEMethod +from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -119,6 +135,158 @@ class TestFp8MoERunnerOwnership(CustomTestCase): self._assert_activation_params_absent(layer) +class TestMxfp4MoEMethodContract(CustomTestCase): + def setUp(self): + self.stack = self.enterContext(ExitStack()) + self.stack.enter_context( + get_context().override_server_args(model_path="dummy", device="cpu") + ) + self.stack.enter_context( + get_flags().moe.override(a2a_backend=MoeA2ABackend.NONE) + ) + self.stack.enter_context( + get_parallel().override( + tp_size=1, + tp_rank=0, + attn_tp_size=1, + attn_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + ) + ) + # Keep real method selection, constructors and layer policy binding; + # replace expert storage and native runner construction with CPU fixtures. + adapters = ( + cutlass_mxfp4.Mxfp4FlashinferCutlassMoEMethod, + Mxfp4FlashinferTrtllmMoEMethod, + Mxfp4HummingMoEMethod, + Mxfp4MarlinMoEMethod, + ) + for adapter in adapters: + self.stack.enter_context( + patch.object( + adapter, "create_weights", staticmethod(self._create_cpu_weights) + ) + ) + for module in ( + "sglang.srt.layers.moe.moe_runner", + "sglang.srt.layers.moe.moe_runner.runner", + ): + self.stack.enter_context( + patch(f"{module}.MoeRunner", side_effect=self._make_runner) + ) + self.stack.enter_context( + patch.object(cutlass_mxfp4, "is_flashinfer_available", return_value=True) + ) + self.stack.enter_context( + patch.object( + Fp8MoEMethod, + "get_moe_routing_method_type", + side_effect=AssertionError("The borrowed FP8 loader owns no runner"), + ) + ) + + @staticmethod + def _create_cpu_weights(layer, num_experts, **kwargs): + layer.w13_weight = torch.empty((num_experts, 1), device="cpu") + + @staticmethod + def _make_runner(runner_backend, config): + return SimpleNamespace(runner_backend=runner_backend, config=config) + + @staticmethod + def _fp4_expert_config(): + return Fp8Config( + is_checkpoint_fp8_serialized=True, + weight_block_size=[128, 128], + is_fp4_experts=True, + ) + + def _check_construction(self, config, backend, platform, expected_type, prefix): + with ( + get_flags().moe.override(runner_backend=backend), + patch.object(fp8_module, "get_platform", return_value=platform), + patch.object(cutlass_mxfp4, "get_platform", return_value=platform), + ): + for requested in ( + None, + RoutingMethodType.Default, + RoutingMethodType.DeepSeekV3, + ): + with self.subTest(requested=requested): + layer = FusedMoE( + num_experts=2, + hidden_size=128, + intermediate_size=128, + layer_id=0, + top_k=1, + quant_config=config, + prefix=prefix, + routing_method_type=requested, + ) + self.assertIs(type(layer.quant_method), expected_type) + self.assertIs(layer.routing_method_type, requested) + self.assertIs( + layer.moe_runner_config.routing_method_type, requested + ) + self.assertEqual(layer.runner_backend, backend) + self.assertIs(layer.quant_method._fp8.quant_config, config) + self.assertFalse(layer.quant_method._fp8._owns_moe_runner) + + def test_fp4_expert_adapters_preserve_routing_policy(self): + """Each selected MXFP4 adapter must construct without borrowing its FP8 + loader's routing policy, including an explicit zero-valued policy.""" + cases = ( + (MoeRunnerBackend.MARLIN, False, False, Mxfp4MarlinMoEMethod), + (MoeRunnerBackend.HUMMING, False, False, Mxfp4HummingMoEMethod), + ( + MoeRunnerBackend.FLASHINFER_MXFP4, + True, + False, + cutlass_mxfp4.Mxfp4FlashinferCutlassMoEMethod, + ), + ( + MoeRunnerBackend.FLASHINFER_MXFP4, + False, + True, + cutlass_mxfp4.Mxfp4FlashinferCutlassMoEMethod, + ), + ( + MoeRunnerBackend.FLASHINFER_MXFP4, + False, + False, + Mxfp4FlashinferTrtllmMoEMethod, + ), + ) + for backend, sm90, sm120, expected_type in cases: + with self.subTest(backend=backend, sm90=sm90, sm120=sm120): + self._check_construction( + config=self._fp4_expert_config(), + backend=backend, + platform=SimpleNamespace(is_sm90=sm90, is_sm120=sm120), + expected_type=expected_type, + prefix="model.layers.0.mlp.experts", + ) + + def test_hybrid_mtp_adapter_preserves_routing_policy(self): + """Excluded MXFP4 MTP experts retain the same construction contract as + adapters selected directly by Fp8Config.""" + prefix = "model.mtp.experts" + config = HybridFp8NvFp4Config( + self._fp4_expert_config(), + ModelOptFp4Config(exclude_modules=[prefix]), + ) + self._check_construction( + config=config, + backend=MoeRunnerBackend.FLASHINFER_MXFP4, + platform=SimpleNamespace(is_sm90=False, is_sm120=False), + expected_type=Mxfp4FlashinferTrtllmMoEMethod, + prefix=prefix, + ) + + class TestFp8MoEAiterQuantInfo(CustomTestCase): """maybe_get_hip_aiter_quant_info assembles what the AITER runner consumes. diff --git a/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_dispatch.py b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_dispatch.py index 84a6213351..9631d3cef7 100644 --- a/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_dispatch.py +++ b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_dispatch.py @@ -367,7 +367,9 @@ class TestNvFp4MoeDispatch(CustomTestCase): (None, 2), ): with self.subTest(policy=policy): - layer.routing_method_type = policy + layer.routing_method_type = method.get_moe_routing_method_type( + layer, policy + ) self.assertEqual( method.apply( layer, SimpleNamespace(hidden_states=torch.zeros((1, 4))) diff --git a/test/registered/unit/models/test_native_moe_routing_policy.py b/test/registered/unit/models/test_native_moe_routing_policy.py index c8276ee6fa..8ee7338dc0 100644 --- a/test/registered/unit/models/test_native_moe_routing_policy.py +++ b/test/registered/unit/models/test_native_moe_routing_policy.py @@ -126,12 +126,76 @@ class TestNativeMoeRoutingPolicy(CustomTestCase): quant_config=self.quant_config, ) + def _implicit_bailing_policy( + self, kind, *, grouped=True, bias=True, renormalize=True, block_routing=False + ): + from sglang.srt.models.bailing_moe_linear import BailingMoE as LinearBailingMoE + from sglang.srt.models.bailing_moe_v3 import BailingMoE + from sglang.srt.models.llada2 import LLaDA2MoeSparseMoeBlock + + factory = { + "v3": BailingMoE, + "linear": LinearBailingMoE, + "llada2": LLaDA2MoeSparseMoeBlock, + }[kind] + config = SimpleNamespace( + num_experts=8, + num_experts_per_tok=2, + num_shared_experts=None if kind == "llada2" else 0, + hidden_size=4, + moe_intermediate_size=128, + hidden_act="silu", + n_group=2 if grouped else 0, + topk_group=1 if grouped else 0, + norm_topk_prob=renormalize, + routed_scaling_factor=2.5, + moe_router_enable_expert_bias=bias, + score_function="sigmoid" if bias else "softmax", + expert_capacity=4 if block_routing else None, + block_size=2, + ) + result = factory(config=config, layer_id=0, quant_config=self.quant_config) + if bias: + result.gate.expert_bias.data.copy_(self.bias) + return result + + def test_implicit_fp8_policies_reach_the_actual_model_bindings(self): + """Existing FP8 defaults remain native only for equivalent model routers.""" + for kind in ("v3", "linear", "llada2"): + with self.subTest(kind=kind): + block = self._implicit_bailing_policy(kind) + output = self._native_output(block) + scores = self.logits[:, 4:6].sigmoid() + weights = scores / scores.sum(-1, keepdim=True) * 2.5 + torch.testing.assert_close( + output, + self._weighted_experts(torch.tensor([[4, 5]]), weights, output), + ) + for grouped, bias, renormalize in ( + (False, False, True), + (True, False, True), + (True, True, False), + ): + with self.subTest( + kind=kind, grouped=grouped, bias=bias, renormalize=renormalize + ): + block = self._implicit_bailing_policy( + kind, grouped=grouped, bias=bias, renormalize=renormalize + ) + self.assertEqual( + block.topk.topk_config.output_format, TopKOutputFormat.STANDARD + ) + with self.assertRaisesRegex( + ValueError, "block routing does not support.*flashinfer_trtllm" + ): + self._implicit_bailing_policy("llada2", block_routing=True) + @staticmethod def _weighted_experts(ids, weights, hidden_states): values = ids.float() + 1 return (values * weights).sum(dim=-1, keepdim=True).expand_as(hidden_states) - def _native_output(self, block): + def _native_output(self, block, *, block_quant=True): from sglang.srt.layers.moe.moe_runner import flashinfer_trtllm hidden_states = torch.ones((1, 4)) @@ -142,10 +206,15 @@ class TestNativeMoeRoutingPolicy(CustomTestCase): policy = block.experts.routing_method_type self.assertEqual(block.experts.moe_runner_config.routing_method_type, policy) - def native_kernel(*, routing_logits, routing_method_type, output, **kwargs): + def native_kernel( + *, routing_logits, routing_method_type, output=None, **kwargs + ): self.assertEqual(routing_method_type, policy) + if not block_quant: + self.assertEqual(routing_logits.dtype, torch.bfloat16) + self.assertFalse(kwargs["use_routing_scales_on_input"]) if routing_method_type == RoutingMethodType.DeepSeekV3: - scores = routing_logits.sigmoid() + scores = routing_logits.float().sigmoid() choices = scores + kwargs["routing_bias"] group_scores = choices.reshape(1, 2, 4).topk(2, dim=-1).values.sum(-1) selected = group_scores.argmax(-1).item() @@ -153,15 +222,18 @@ class TestNativeMoeRoutingPolicy(CustomTestCase): ids = ids + selected * 4 weights = scores.gather(1, ids) elif routing_method_type == RoutingMethodType.SigmoidRenorm: - weights, ids = routing_logits.sigmoid().topk(2) + weights, ids = routing_logits.float().sigmoid().topk(2) else: self.assertEqual( routing_method_type, RoutingMethodType.RenormalizeNaive ) - weights, ids = routing_logits.softmax(-1).topk(2) + weights, ids = routing_logits.float().softmax(-1).topk(2) weights = weights / weights.sum(-1, keepdim=True) weights = weights * kwargs["routed_scaling_factor"] - output.copy_(self._weighted_experts(ids, weights, hidden_states)) + result = self._weighted_experts(ids, weights, hidden_states) + if output is None: + return result.to(torch.bfloat16) + output.copy_(result) quant_info = flashinfer_trtllm.FlashInferTrtllmFp8MoeQuantInfo( w13_weight=torch.empty(8, 256, 4), @@ -171,10 +243,14 @@ class TestNativeMoeRoutingPolicy(CustomTestCase): local_num_experts=8, intermediate_size=128, routing_method_type=policy, - block_quant=True, + block_quant=block_quant, weight_block_k=128, w13_weight_scale_inv=torch.ones(8, 2, 1), w2_weight_scale_inv=torch.ones(8, 1, 1), + w13_input_scale=torch.ones(1), + output1_scales_scalar=torch.ones(8), + output1_scales_gate_scalar=torch.ones(8), + output2_scales_scalar=torch.ones(8), ) provider = ModuleType("flashinfer.fused_moe") provider.Fp8QuantizationType = SimpleNamespace(MxFp8=1, DeepSeekFp8=0) @@ -190,6 +266,16 @@ class TestNativeMoeRoutingPolicy(CustomTestCase): "trtllm_fp8_block_scale_moe_out_wrapper", side_effect=native_kernel, ), + patch.object( + flashinfer_trtllm, + "scaled_fp8_quant", + return_value=(hidden_states, torch.ones(1)), + ), + patch.object( + flashinfer_trtllm, + "trtllm_fp8_per_tensor_scale_moe_wrapper", + side_effect=native_kernel, + ), ): return flashinfer_trtllm.fused_experts_none_to_flashinfer_trtllm_fp8( StandardDispatchOutput(hidden_states, None, routing), @@ -197,6 +283,17 @@ class TestNativeMoeRoutingPolicy(CustomTestCase): block.experts.moe_runner_config, ).hidden_states + def test_unpadded_per_tensor_fp8_keeps_the_logits_policy_and_scale(self): + """The routed adapter must not alter an admitted logits-only invocation.""" + block = self._implicit_bailing_policy("v3") + output = self._native_output(block, block_quant=False) + scores = self.logits[:, 4:6].sigmoid() + weights = scores / scores.sum(-1, keepdim=True) * 2.5 + expected = self._weighted_experts(torch.tensor([[4, 5]]), weights, output).to( + torch.bfloat16 + ) + torch.testing.assert_close(output, expected, rtol=0, atol=0) + def test_mimo_grouped_bias_retains_native_selection_and_weights(self): """Equivalent grouped routing must not incur materialized dispatch.""" block = self._mimo()