[Fix] moe: Preserve CUTLASS routing metadata and execution budgets (#45917)

This commit is contained in:
Cheng Wan
2026-10-23 12:32:05 -07:00
committed by GitHub
parent 8b9b992e41
commit ac3db731b3
32 changed files with 3221 additions and 206 deletions
@@ -0,0 +1,194 @@
#pragma once
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.cuh>
#include <tvm/ffi/container/tensor.h>
#include <tvm/ffi/optional.h>
#include <algorithm>
#include <cstdint>
#include <limits>
namespace sglang {
namespace cutlass_moe {
template <bool kTransposeProblemSizes>
__global__ void count_expert_routes(
const int32_t* __restrict__ topk_ids,
int32_t* __restrict__ problem_sizes1,
int32_t* __restrict__ problem_sizes2,
int32_t num_routes,
int32_t n,
int32_t k) {
__shared__ int32_t count;
if (threadIdx.x == 0) count = 0;
__syncthreads();
const int32_t expert = blockIdx.x;
int32_t local_count = 0;
for (uint32_t i = threadIdx.x; i < static_cast<uint32_t>(num_routes); i += blockDim.x) {
local_count += topk_ids[i] == expert;
}
atomicAdd(&count, local_count);
__syncthreads();
if (threadIdx.x == 0) {
constexpr int kCountColumn = kTransposeProblemSizes ? 1 : 0;
constexpr int kOutputColumn = 1 - kCountColumn;
problem_sizes1[expert * 3 + kCountColumn] = count;
problem_sizes1[expert * 3 + kOutputColumn] = 2 * n;
problem_sizes1[expert * 3 + 2] = k;
problem_sizes2[expert * 3 + kCountColumn] = count;
problem_sizes2[expert * 3 + kOutputColumn] = k;
problem_sizes2[expert * 3 + 2] = n;
}
}
template <bool kTransposeProblemSizes>
__global__ void compute_expert_offsets(
const int32_t* __restrict__ problem_sizes1,
int32_t* __restrict__ expert_offsets,
int32_t* __restrict__ blockscale_offsets,
int32_t num_experts) {
int32_t offset = 0;
int32_t blockscale_offset = 0;
expert_offsets[0] = 0;
if (blockscale_offsets != nullptr) blockscale_offsets[0] = 0;
for (int32_t expert = 0; expert < num_experts; ++expert) {
constexpr int kCountColumn = kTransposeProblemSizes ? 1 : 0;
const int32_t count = problem_sizes1[expert * 3 + kCountColumn];
offset += count;
expert_offsets[expert + 1] = offset;
if (blockscale_offsets != nullptr) {
blockscale_offset += (count + 127) / 128 * 128;
blockscale_offsets[expert + 1] = blockscale_offset;
}
}
}
__global__ void complete_route_maps(
const int32_t* __restrict__ topk_ids,
const int32_t* __restrict__ expert_offsets,
int32_t* __restrict__ input_permutation,
int32_t* __restrict__ output_permutation,
int32_t num_routes,
int32_t topk,
int32_t num_experts) {
const int32_t expert = blockIdx.x;
// Exactly one CTA owns each expert's interval, so its cursor is block-local.
__shared__ int32_t cursor;
if (threadIdx.x == 0) cursor = expert_offsets[expert];
__syncthreads();
for (uint32_t i = threadIdx.x; i < static_cast<uint32_t>(num_routes); i += blockDim.x) {
const int32_t route = topk_ids[i];
if (route == expert) {
const int32_t destination = atomicAdd(&cursor, 1);
input_permutation[destination] = i / topk;
output_permutation[i] = destination;
} else if (expert == 0 && (route < 0 || route >= num_experts)) {
output_permutation[i] = -1;
}
}
// The grouped GEMMs use only [0, live); full-capacity gathers also read the tail.
const int32_t live = expert_offsets[num_experts];
const int64_t thread = static_cast<int64_t>(expert) * blockDim.x + threadIdx.x;
const int64_t stride = static_cast<int64_t>(num_experts) * blockDim.x;
for (int64_t i = live + thread; i < num_routes; i += stride) {
input_permutation[i] = 0;
}
}
} // namespace cutlass_moe
/// \brief Prepare CUTLASS grouped-GEMM shapes and complete route maps.
/// \param input_permutation Destination-to-token map, with zero-filled unused capacity.
/// \param output_permutation Route-to-destination map, with -1 for ids outside [0, num_experts).
/// \param blockscale_offsets Optional MXFP8 row offsets, rounded up to 128 per expert.
template <bool kTransposeProblemSizes = false>
inline void cutlass_prepare_moe_input(
tvm::ffi::TensorView topk_ids,
tvm::ffi::TensorView expert_offsets,
tvm::ffi::TensorView problem_sizes1,
tvm::ffi::TensorView problem_sizes2,
tvm::ffi::TensorView input_permutation,
tvm::ffi::TensorView output_permutation,
int64_t num_experts,
int64_t n,
int64_t k,
tvm::ffi::Optional<tvm::ffi::TensorView> blockscale_offsets) {
using namespace host;
constexpr int64_t max_index = std::numeric_limits<int32_t>::max();
CHECK_HOST(num_experts > 0 && num_experts <= max_index / 3);
CHECK_HOST(n > 0 && n <= max_index / 2 && k > 0 && k <= max_index);
auto device = SymbolicDevice{};
auto tokens = SymbolicSize{"tokens"};
auto topk = SymbolicSize{"topk"};
TensorMatcher({tokens, topk})
.with_strides({-1, -1})
.with_dtype<int32_t>()
.with_device<kDLCUDA>(device)
.verify(topk_ids);
const int64_t num_routes = topk_ids.numel();
// Empty PyTorch dimensions can have noncanonical strides; no ID is read.
RuntimeCheck(num_routes == 0 || topk_ids.is_contiguous(), "topk_ids must be contiguous when nonempty");
CHECK_HOST(num_routes <= max_index);
TensorMatcher({num_experts + 1}).with_dtype<int32_t>().with_device(device).verify(expert_offsets);
TensorMatcher({num_experts, 3})
.with_dtype<int32_t>()
.with_device(device)
.verify(problem_sizes1)
.verify(problem_sizes2);
TensorMatcher({num_routes})
.with_dtype<int32_t>()
.with_device(device)
.verify(input_permutation)
.verify(output_permutation);
int32_t* blockscale_ptr = nullptr;
if (blockscale_offsets.has_value()) {
TensorMatcher({num_experts + 1}).with_dtype<int32_t>().with_device(device).verify(blockscale_offsets.value());
CHECK_HOST(num_routes + 127 * std::min(num_experts, num_routes) <= max_index);
blockscale_ptr = static_cast<int32_t*>(blockscale_offsets.value().data_ptr());
}
const auto threads = static_cast<uint32_t>(std::min<int64_t>(512, std::max<int64_t>(1, num_routes)));
const auto experts = static_cast<uint32_t>(num_experts);
const auto routes = static_cast<int32_t>(num_routes);
const auto* ids = static_cast<const int32_t*>(topk_ids.data_ptr());
auto* sizes1 = static_cast<int32_t*>(problem_sizes1.data_ptr());
auto* sizes2 = static_cast<int32_t*>(problem_sizes2.data_ptr());
auto* offsets = static_cast<int32_t*>(expert_offsets.data_ptr());
const auto stream = LaunchKernel::resolve_device(device.unwrap());
LaunchKernel(dim3(experts), dim3(threads), stream)(
cutlass_moe::count_expert_routes<kTransposeProblemSizes>,
ids,
sizes1,
sizes2,
routes,
static_cast<int32_t>(n),
static_cast<int32_t>(k));
LaunchKernel(dim3(1), dim3(1), stream)(
cutlass_moe::compute_expert_offsets<kTransposeProblemSizes>,
sizes1,
offsets,
blockscale_ptr,
static_cast<int32_t>(num_experts));
if (num_routes != 0) {
LaunchKernel(dim3(experts), dim3(threads), stream)(
cutlass_moe::complete_route_maps,
ids,
offsets,
static_cast<int32_t*>(input_permutation.data_ptr()),
static_cast<int32_t*>(output_permutation.data_ptr()),
routes,
static_cast<int32_t>(topk.unwrap()),
static_cast<int32_t>(num_experts));
}
}
} // namespace sglang
@@ -0,0 +1,142 @@
#pragma once
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <cstdint>
#include <limits>
#include <type_traits>
namespace sglang {
namespace shuffle_weighted_sum {
SGL_DEVICE float accumulate(float value, float factor, float total) {
float result;
// Keep the slot-order FMA, including +0 * +0 for inactive slots. No FTZ.
asm volatile("fma.rn.f32 %0, %1, %2, %3;" : "=f"(result) : "f"(value), "f"(factor), "f"(total));
return result;
}
template <
typename scalar_t,
typename weight_t,
typename index_t,
int64_t kTopK,
int kThreads,
bool kCastWeightsToInputDType>
__global__ void reduce(
const scalar_t* inputs,
scalar_t* output,
const index_t* permutation,
const weight_t* weights,
int64_t width,
int64_t rows,
int64_t input_stride,
int64_t output_stride,
int64_t weight_stride0,
int64_t weight_stride1) {
constexpr int kItems = 2;
const int64_t token = blockIdx.x;
const int64_t first_column = static_cast<int64_t>(blockIdx.y) * kThreads * kItems + threadIdx.x;
if (first_column >= width) return;
float totals[kItems] = {0.0f, 0.0f};
for (int64_t slot = 0; slot < kTopK; ++slot) {
const int64_t row = permutation[token * kTopK + slot];
const bool active = row >= 0 && row < rows;
float factor = 0.0f;
if (active) {
factor = device::cast<float>(weights[token * weight_stride0 + slot * weight_stride1]);
if constexpr (kCastWeightsToInputDType && !std::is_same_v<scalar_t, weight_t>) {
// Match scores.to(inputs.dtype) before the FP32 slot-order FMA.
factor = device::cast<float>(device::cast<scalar_t>(factor));
}
}
#pragma unroll
for (int item = 0; item < kItems; ++item) {
const int64_t column = first_column + item * kThreads;
float value = 0.0f;
if (active && column < width) value = device::cast<float>(inputs[row * input_stride + column]);
totals[item] = accumulate(value, factor, totals[item]);
}
}
#pragma unroll
for (int item = 0; item < kItems; ++item) {
const int64_t column = first_column + item * kThreads;
if (column < width) output[token * output_stride + column] = device::cast<scalar_t>(totals[item]);
}
}
} // namespace shuffle_weighted_sum
/// \brief Reduce mapped rows in slot order without reading inactive payloads/scores.
/// \param permutation Contiguous int32/int64 map of any rank, one entry per weight.
/// \param weights Potentially strided scores; rows outside the input extent are inactive.
template <
typename scalar_t,
typename weight_t,
typename index_t,
int64_t kTopK,
int kThreads,
bool kCastWeightsToInputDType>
void shuffle_weighted_sum_cuda(
tvm::ffi::TensorView inputs,
tvm::ffi::TensorView output,
tvm::ffi::TensorView permutation,
tvm::ffi::TensorView weights) {
using namespace host;
static_assert(std::is_same_v<scalar_t, fp16_t> || std::is_same_v<scalar_t, bf16_t>);
static_assert(
std::is_same_v<weight_t, fp16_t> || std::is_same_v<weight_t, bf16_t> || std::is_same_v<weight_t, float>);
static_assert(std::is_same_v<index_t, int32_t> || std::is_same_v<index_t, int64_t>);
static_assert(kTopK >= 0 && kThreads > 0 && kThreads <= 1024);
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
auto rows = SymbolicSize{"input_rows"};
auto tokens = SymbolicSize{"output_rows"};
auto width = SymbolicSize{"width"};
auto input_stride = SymbolicSize{"input_stride"};
auto output_stride = SymbolicSize{"output_stride"};
auto weight_stride0 = SymbolicSize{"weight_stride0"};
auto weight_stride1 = SymbolicSize{"weight_stride1"};
TensorMatcher({rows, width})
.with_strides({input_stride, 1})
.with_dtype<scalar_t>()
.with_device(device)
.verify(inputs);
TensorMatcher({tokens, width})
.with_strides({output_stride, 1})
.with_dtype<scalar_t>()
.with_device(device)
.verify(output);
TensorMatcher({tokens, kTopK})
.with_strides({weight_stride0, weight_stride1})
.with_dtype<weight_t>()
.with_device(device)
.verify(weights);
auto map_dtype = SymbolicDType{};
map_dtype.set_options<index_t>();
map_dtype.verify(permutation.dtype());
device.verify(permutation.device());
CHECK_HOST(permutation.is_contiguous() && permutation.numel() == weights.numel());
if (tokens.unwrap() == 0 || width.unwrap() == 0) return;
constexpr int64_t kTile = kThreads * 2;
const int64_t tiles = div_ceil(width.unwrap(), kTile);
// These are the same x/y grid limits as the Triton fallback.
CHECK_HOST(tokens.unwrap() <= std::numeric_limits<int32_t>::max() && tiles <= 65535);
LaunchKernel(dim3(static_cast<uint32_t>(tokens.unwrap()), static_cast<uint32_t>(tiles)), kThreads, device.unwrap())(
shuffle_weighted_sum::reduce<scalar_t, weight_t, index_t, kTopK, kThreads, kCastWeightsToInputDType>,
static_cast<const scalar_t*>(inputs.data_ptr()),
static_cast<scalar_t*>(output.data_ptr()),
static_cast<const index_t*>(permutation.data_ptr()),
static_cast<const weight_t*>(weights.data_ptr()),
width.unwrap(),
rows.unwrap(),
input_stride.unwrap(),
output_stride.unwrap(),
weight_stride0.unwrap(),
weight_stride1.unwrap());
}
} // namespace sglang
+19 -1
View File
@@ -71,10 +71,19 @@ register_kernel(
)
register_kernel(
KernelSpec(
op="moe.cutlass_prepare_moe_input",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.moe.cutlass_prepare_moe_input:cutlass_prepare_moe_input",
capabilities=CUDA_ONLY,
)
)
register_kernel(
KernelSpec(
op="moe.shuffle_weighted_sum",
backend=KernelBackend.TRITON,
backend=KernelBackend.JIT,
target="sglang.kernels.ops.moe.shuffle_weighted_sum:shuffle_weighted_sum",
capabilities=CUDA_ONLY,
)
@@ -112,6 +121,15 @@ register_kernel(
)
)
register_kernel(
KernelSpec(
op="moe.compute_cutlass_w4a8_moe_input_scale",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.moe.ep_moe_kernels:compute_cutlass_w4a8_moe_input_scale",
capabilities=CUDA_ONLY,
)
)
def moe_align_block_size(
topk_ids: torch.Tensor,
@@ -0,0 +1,72 @@
"""Complete routing maps and grouped-GEMM shapes for CUTLASS MoE."""
from __future__ import annotations
from typing import TYPE_CHECKING, Optional
import torch
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
from sglang.srt.utils.custom_op import register_custom_op
if TYPE_CHECKING:
from tvm_ffi.module import Module
@cache_once
def _jit_cutlass_prepare_moe_input_module(
transpose_problem_sizes: bool = False,
) -> Module:
args = make_cpp_args(transpose_problem_sizes)
return load_jit(
"cutlass_prepare_moe_input",
*args,
cuda_files=["moe/cutlass_prepare_moe_input.cuh"],
cuda_wrappers=[("prepare", f"cutlass_prepare_moe_input<{args}>")],
)
@register_custom_op(
op_name="cutlass_prepare_moe_input",
mutates_args=[
"expert_offsets",
"problem_sizes1",
"problem_sizes2",
"input_permutation",
"output_permutation",
"blockscale_offsets",
],
)
def cutlass_prepare_moe_input(
topk_ids: torch.Tensor,
expert_offsets: torch.Tensor,
problem_sizes1: torch.Tensor,
problem_sizes2: torch.Tensor,
input_permutation: torch.Tensor,
output_permutation: torch.Tensor,
num_experts: int,
n: int,
k: int,
blockscale_offsets: Optional[torch.Tensor] = None,
*,
transpose_problem_sizes: bool = False,
) -> None:
"""Write contiguous int32 CUTLASS metadata, including every unused map entry.
Routes outside [0, num_experts) have output permutation -1. Input-map
capacity beyond expert_offsets[-1] points at token zero for full gathers.
Optional blockscale offsets count per-expert rows rounded up to 128.
Transposed problem sizes put the token count in N instead of M.
"""
_jit_cutlass_prepare_moe_input_module(transpose_problem_sizes).prepare(
topk_ids,
expert_offsets,
problem_sizes1,
problem_sizes2,
input_permutation,
output_permutation,
num_experts,
n,
k,
blockscale_offsets,
)
@@ -145,13 +145,27 @@ def deepep_post_reorder_triton_kernel(
@triton.jit
def compute_src2dst_triton_kernel(
reorder_ids, src2dst, num_toks, BLOCK_SIZE: tl.constexpr
sorted_topk_ids, reorder_ids, src2dst, num_toks, BLOCK_SIZE: tl.constexpr
):
first_nonnegative = 0
if tl.load(sorted_topk_ids) < 0:
left, right = 0, num_toks
while left < right:
middle = left + (right - left) // 2
expert_id = tl.load(sorted_topk_ids + middle)
if expert_id < 0:
left = middle + 1
else:
right = middle
first_nonnegative = left
pid = tl.program_id(axis=0)
dst_id = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = dst_id < num_toks
src_id = tl.load(reorder_ids + dst_id, mask=mask)
tl.store(src2dst + src_id, dst_id, mask=mask)
# Native expert offsets omit negative IDs, so packed rows must omit that prefix.
packed_id = tl.where(dst_id >= first_nonnegative, dst_id - first_nonnegative, -1)
tl.store(src2dst + src_id, packed_id, mask=mask)
@triton.jit
@@ -189,18 +203,95 @@ def deepep_run_moe_deep_preprocess(topk_ids: torch.Tensor, num_experts: int):
def cutlass_w4_run_moe_ep_preproess(topk_ids: torch.Tensor):
_, reorder_ids = torch.sort(topk_ids.view(-1), stable=True)
src2dst = torch.empty(topk_ids.numel(), device=topk_ids.device, dtype=torch.int32)
if topk_ids.numel() == 0:
return src2dst
sorted_topk_ids, reorder_ids = torch.sort(topk_ids.view(-1), stable=True)
BLOCK_SIZE = 512
grid = (triton.cdiv(topk_ids.numel(), BLOCK_SIZE),)
src2dst = torch.empty(topk_ids.numel(), device=topk_ids.device, dtype=torch.int32)
compute_src2dst_triton_kernel[grid](
reorder_ids, src2dst, topk_ids.numel(), BLOCK_SIZE
sorted_topk_ids, reorder_ids, src2dst, topk_ids.numel(), BLOCK_SIZE
)
return src2dst
@triton.jit
def _compute_cutlass_w4a8_moe_input_scale_kernel(
input_ptr,
topk_ids_ptr,
output_s_ptr,
stride_input_row,
stride_input_col,
stride_ids_row,
stride_ids_col,
hidden_size: tl.constexpr,
topk: tl.constexpr,
num_local_experts: tl.constexpr,
FP8_MAX: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
routes = tl.arange(0, BLOCK_TOPK)
expert_ids = tl.load(
topk_ids_ptr + row * stride_ids_row + routes * stride_ids_col,
mask=routes < topk,
other=-1,
)
local_route = (expert_ids >= 0) & (expert_ids < num_local_experts)
active = tl.sum(local_route.to(tl.int32), 0) > 0
columns = tl.arange(0, BLOCK_SIZE)
values = tl.load(
input_ptr + row * stride_input_row + columns * stride_input_col,
mask=(columns < hidden_size) & active,
other=0.0,
).to(tl.float32)
absmax = tl.max(tl.abs(values), 0)
tl.atomic_max(output_s_ptr, absmax / FP8_MAX, sem="relaxed")
def compute_cutlass_w4a8_moe_input_scale(
input: torch.Tensor,
topk_ids: torch.Tensor,
output_s: torch.Tensor,
num_local_experts: int,
):
"""Accumulate the FP8 scale from rows with a local expert route.
The caller initializes output_s to its minimum finite scale. Inputs and IDs
are preserved, and fully inactive rows never load activations, including NaNs.
"""
assert input.ndim == 2
assert input.dtype in (torch.float16, torch.bfloat16, torch.float32)
assert topk_ids.ndim == 2 and topk_ids.shape[0] == input.shape[0]
assert topk_ids.dtype in (torch.int32, torch.int64)
assert output_s.numel() == 1 and output_s.dtype == torch.float32
assert input.is_cuda
assert input.device == topk_ids.device == output_s.device
if input.numel() == 0 or topk_ids.shape[1] == 0:
return
hidden_size = input.shape[1]
block_size = triton.next_power_of_2(max(1, hidden_size))
_compute_cutlass_w4a8_moe_input_scale_kernel[(input.shape[0],)](
input,
topk_ids,
output_s,
input.stride(0),
input.stride(1),
topk_ids.stride(0),
topk_ids.stride(1),
hidden_size=hidden_size,
topk=topk_ids.shape[1],
num_local_experts=num_local_experts,
FP8_MAX=torch.finfo(torch.float8_e4m3fn).max,
BLOCK_SIZE=block_size,
BLOCK_TOPK=triton.next_power_of_2(topk_ids.shape[1]),
num_warps=4 if block_size < 4096 else 8,
)
@triton.jit
def pre_reorder_triton_kernel_for_cutlass_moe(
input_ptr,
@@ -241,7 +332,7 @@ def pre_reorder_triton_kernel_for_cutlass_moe(
out_data = (in_data * a1_scale).to(OutDtype)
for idx in range(topk):
expert_id = tl.load(token_topk_ids_ptr + idx)
if expert_id != num_local_experts:
if (expert_id >= 0) & (expert_id < num_local_experts):
dst_idx = tl.load(token_src2dst_ptr + idx).to(tl.int64)
tl.store(dst_ptr_offs + dst_idx * hidden_size, out_data, mask=mask)
@@ -855,7 +946,7 @@ def post_reorder_triton_kernel_for_cutlass_moe(
sum_vec = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
for idx in range(topk):
expert_id = tl.load(token_topk_ids_ptr + idx)
if expert_id != num_local_experts:
if (expert_id >= 0) & (expert_id < num_local_experts):
dst_idx_int32 = tl.load(token_src2dst_ptr + idx)
dst_idx = dst_idx_int32.to(tl.int64)
dst_idx = dst_idx
@@ -1,9 +1,56 @@
"""Reduce routed rows while skipping inactive permutation entries."""
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
import triton
import triton.language as tl
from sglang.kernels.jit.utils import (
cache_once,
is_hip_runtime,
is_musa_runtime,
load_jit,
make_cpp_args,
)
from sglang.kernels.jit.utils.common import is_sail_runtime
from sglang.srt.utils.custom_op import register_custom_op
if TYPE_CHECKING:
from tvm_ffi.module import Module
@cache_once
def _jit_shuffle_weighted_sum_module(
dtype: torch.dtype,
output_dtype: torch.dtype,
weight_dtype: torch.dtype,
index_dtype: torch.dtype,
topk: int,
cast_weights_to_input_dtype: bool = False,
) -> Module | None:
if (
dtype not in (torch.float16, torch.bfloat16)
or output_dtype != dtype
or weight_dtype not in (torch.float16, torch.bfloat16, torch.float32)
or index_dtype not in (torch.int32, torch.int64)
or is_hip_runtime()
or is_musa_runtime()
or is_sail_runtime()
):
return None
args = make_cpp_args(
dtype, weight_dtype, index_dtype, topk, 128, cast_weights_to_input_dtype
)
return load_jit(
"shuffle_weighted_sum",
*args,
cuda_files=["moe/shuffle_weighted_sum.cuh"],
cuda_wrappers=[("reduce", f"shuffle_weighted_sum_cuda<{args}>")],
)
@triton.jit(do_not_specialize=["rows", "weight_stride0", "weight_stride1"])
def _shuffle_weighted_sum_kernel(
@@ -19,6 +66,7 @@ def _shuffle_weighted_sum_kernel(
weight_stride0,
weight_stride1,
BLOCK: tl.constexpr,
CAST_WEIGHTS_TO_INPUT_DTYPE: tl.constexpr = False,
):
token = tl.program_id(0)
columns = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
@@ -35,7 +83,10 @@ def _shuffle_weighted_sum_kernel(
weights + token * weight_stride0 + slot * weight_stride1,
mask=active,
other=0,
).to(tl.float32)
)
if CAST_WEIGHTS_TO_INPUT_DTYPE:
factor = factor.to(inputs.dtype.element_ty)
factor = factor.to(tl.float32)
total += values * factor
tl.store(
output + token.to(tl.int64) * output_stride + columns,
@@ -44,13 +95,15 @@ def _shuffle_weighted_sum_kernel(
)
def shuffle_weighted_sum(
def _shuffle_weighted_sum_triton(
inputs: torch.Tensor,
output: torch.Tensor,
permutation: torch.Tensor,
weights: torch.Tensor,
*,
cast_weights_to_input_dtype: bool = False,
) -> None:
"""Sum mapped expert outputs; inactive maps read neither data nor weights."""
"""Sum mapped rows, optionally rounding scores to the input dtype first."""
assert inputs.ndim == output.ndim == weights.ndim == 2
assert inputs.shape[1] == output.shape[1]
assert inputs.stride(1) == output.stride(1) == 1
@@ -58,6 +111,17 @@ def shuffle_weighted_sum(
assert permutation.is_contiguous() and permutation.numel() == weights.numel()
if output.numel() == 0:
return
if cast_weights_to_input_dtype and (
inputs.dtype not in (torch.float16, torch.bfloat16)
or weights.dtype not in (torch.float16, torch.bfloat16, torch.float32)
or not inputs.is_cuda
or is_hip_runtime()
or is_musa_runtime()
or is_sail_runtime()
):
# Preserve Torch conversion semantics outside the fused CUDA types.
weights = weights.to(inputs.dtype)
cast_weights_to_input_dtype = False
_shuffle_weighted_sum_kernel[(output.shape[0], triton.cdiv(output.shape[1], 256))](
inputs,
output,
@@ -71,4 +135,41 @@ def shuffle_weighted_sum(
weights.stride(0),
weights.stride(1),
BLOCK=256,
CAST_WEIGHTS_TO_INPUT_DTYPE=cast_weights_to_input_dtype,
)
@register_custom_op(op_name="shuffle_weighted_sum", mutates_args=["output"])
def shuffle_weighted_sum(
inputs: torch.Tensor,
output: torch.Tensor,
permutation: torch.Tensor,
weights: torch.Tensor,
*,
cast_weights_to_input_dtype: bool = False,
) -> None:
"""Sum mapped rows, optionally rounding scores to the input dtype first."""
if (
inputs.is_cuda
and inputs.dtype in (torch.float16, torch.bfloat16)
and inputs.ndim == output.ndim == weights.ndim == 2
and output.numel() != 0
):
module = _jit_shuffle_weighted_sum_module(
inputs.dtype,
output.dtype,
weights.dtype,
permutation.dtype,
weights.shape[1],
cast_weights_to_input_dtype,
)
if module is not None:
module.reduce(inputs, output, permutation, weights)
return
_shuffle_weighted_sum_triton(
inputs,
output,
permutation,
weights,
cast_weights_to_input_dtype=cast_weights_to_input_dtype,
)
+17 -7
View File
@@ -14,11 +14,13 @@ if _is_cuda:
es_sm100_mxfp8_blockscaled_grouped_mm,
es_sm100_mxfp8_blockscaled_grouped_quant,
fp8_blockwise_scaled_grouped_mm,
prepare_moe_input,
shuffle_rows,
)
from sglang.kernels.ops.activation.activation import silu_and_mul
from sglang.kernels.ops.moe.cutlass_prepare_moe_input import (
cutlass_prepare_moe_input,
)
from sglang.kernels.ops.moe.shuffle_rows_with_scales import (
shuffle_rows_with_scales,
)
@@ -142,10 +144,6 @@ def cutlass_fused_experts_fp8(
topk = topk_ids.size(1)
device = a.device
# prepare_moe_input writes only live routes; gathers still span the tail.
a_map = torch.zeros((topk_ids.numel()), dtype=torch.int32, device=device)
c_map = torch.full((topk_ids.numel(),), -1, dtype=torch.int32, device=device)
if use_mxfp8:
assert es_up and es_down, "MXFP8 requires expert-specialization for both GEMMs"
assert get_platform().is_sm100, "MXFP8 requires SM100"
@@ -178,13 +176,23 @@ def cutlass_fused_experts_fp8(
(max_total + mxfp8_blockscale_align - 1) // mxfp8_blockscale_align
) * mxfp8_blockscale_align
if m == 0 or topk == 0:
if output is None:
output = torch.empty((m, k), dtype=out_dtype, device=device)
if m != 0:
output.zero_()
return output
a_map = torch.empty((topk_ids.numel(),), dtype=torch.int32, device=device)
c_map = torch.empty((topk_ids.numel(),), dtype=torch.int32, device=device)
blockscale_offsets = None
if use_mxfp8 and (es_up or es_down):
blockscale_offsets = torch.empty(
(num_experts + 1,), dtype=torch.int32, device=device
)
prepare_moe_input(
cutlass_prepare_moe_input(
topk_ids,
expert_offsets,
problem_sizes1,
@@ -346,5 +354,7 @@ def cutlass_fused_experts_fp8(
if output is None:
output = torch.empty((m, k), device=device, dtype=out_dtype)
shuffle_weighted_sum(c2, output, c_map, topk_weights.to(out_dtype))
shuffle_weighted_sum(
c2, output, c_map, topk_weights, cast_weights_to_input_dtype=True
)
return output
@@ -18,10 +18,14 @@ if _is_cuda_alike:
if _is_cuda:
from sglang.kernels.ops.activation.activation import silu_and_mul
from sglang.kernels.ops.moe.cutlass_prepare_moe_input import (
cutlass_prepare_moe_input,
)
else:
from sgl_kernel import silu_and_mul
from sglang.kernels.ops.moe.ep_moe_kernels import (
compute_cutlass_w4a8_moe_input_scale,
cutlass_w4_run_moe_ep_preproess,
deepep_ll_get_cutlass_w4a8_moe_mm_data,
deepep_permute_triton_kernel,
@@ -34,10 +38,7 @@ from sglang.kernels.ops.moe.ep_moe_kernels import (
silu_mul_dynamic_tensorwise_quant_for_cutlass_moe,
silu_mul_static_tensorwise_quant_for_cutlass_moe,
)
from sglang.kernels.ops.quantization.per_tensor_quant_fp8 import (
per_tensor_absmax_fp8,
per_tensor_quant_fp8,
)
from sglang.kernels.ops.quantization.per_tensor_quant_fp8 import per_tensor_quant_fp8
def cutlass_w4a8_moe(
@@ -85,6 +86,7 @@ def cutlass_w4a8_moe(
Shape: [num_experts, N // 512, K * 4]
- topk_weights (torch.Tensor): The weights of each token->expert mapping.
- topk_ids (torch.Tensor): The ids of each token->expert mapping.
Noncontiguous IDs are materialized once for the native flat-pointer consumers.
- a_strides1 (torch.Tensor): The input strides of the first grouped gemm.
- b_strides1 (torch.Tensor): The weights strides of the first grouped gemm.
- c_strides1 (torch.Tensor): The output strides of the first grouped gemm.
@@ -126,13 +128,42 @@ def cutlass_w4a8_moe(
if apply_router_weight_on_input:
assert topk == 1, "apply_router_weight_on_input is only implemented for topk=1"
device = a.device
# Both graph padding and nonlocal routes use -1, including without EP.
topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids)
# Native expert counts and reorder kernels read contiguous int32 IDs.
topk_ids = topk_ids.contiguous()
src2dst = cutlass_w4_run_moe_ep_preproess(
topk_ids,
)
device = a.device
if a1_scale is None:
a1_scale = torch.full(
(1,),
1e-10 / torch.finfo(torch.float8_e4m3fn).max,
dtype=torch.float32,
device=device,
)
compute_cutlass_w4a8_moe_input_scale(
input=a,
topk_ids=topk_ids,
output_s=a1_scale,
num_local_experts=num_local_experts,
)
if _is_cuda:
a_map = torch.empty(topk_ids.numel(), dtype=torch.int32, device=device)
c_map = torch.empty(topk_ids.numel(), dtype=torch.int32, device=device)
cutlass_prepare_moe_input(
topk_ids,
expert_offsets,
problem_sizes1,
problem_sizes2,
a_map,
c_map,
num_local_experts,
n,
k,
transpose_problem_sizes=True,
)
src2dst = c_map
else:
src2dst = cutlass_w4_run_moe_ep_preproess(topk_ids)
gateup_input = torch.empty(
(m * topk, k),
@@ -140,15 +171,6 @@ def cutlass_w4a8_moe(
dtype=torch.float8_e4m3fn,
)
# TODO: fuse per_tensor_absmax_fp8 and pre_reorder_for_cutlass_moe
if a1_scale is None:
a1_scale = torch.zeros(1, dtype=torch.float32, device=device)
# Inactive rows must not change the quantization of live expert inputs.
inactive_rows = (topk_ids == num_local_experts).all(dim=-1, keepdim=True)
scale_input = a.masked_fill(inactive_rows, 0.0)
per_tensor_absmax_fp8(scale_input, a1_scale)
a1_scale.clamp_(min=1e-10 / torch.finfo(torch.float8_e4m3fn).max)
pre_reorder_for_cutlass_moe(
a,
gateup_input,
@@ -161,22 +183,20 @@ def cutlass_w4a8_moe(
k,
)
# NOTE: a_map and c_map are not used in the get_cutlass_w4a8_moe_mm_data kernel,
# they are kept to allow for a quick switch of the permutation logic
# from the current triton kernel implementation to the cutlass-based one if needed.
a_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
c_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
get_cutlass_w4a8_moe_mm_data(
topk_ids,
expert_offsets,
problem_sizes1,
problem_sizes2,
a_map,
c_map,
num_local_experts,
n,
k,
)
if not _is_cuda:
a_map = torch.empty(topk_ids.numel(), dtype=torch.int32, device=device)
c_map = torch.empty(topk_ids.numel(), dtype=torch.int32, device=device)
get_cutlass_w4a8_moe_mm_data(
topk_ids,
expert_offsets,
problem_sizes1,
problem_sizes2,
a_map,
c_map,
num_local_experts,
n,
k,
)
c1 = torch.empty((m * topk, n * 2), device=device, dtype=torch.bfloat16)
c2 = torch.empty((m * topk, k), device=device, dtype=torch.bfloat16)
@@ -323,3 +323,63 @@ def trtllm_fp8_per_tensor_scale_moe_wrapper(
kwargs["activation_type"] = ActivationType(activation_type)
return trtllm_fp8_per_tensor_scale_moe(**kwargs)
@register_custom_op(mutates_args=["output"])
def trtllm_fp8_per_tensor_scale_routed_moe_out_wrapper(
topk_ids: torch.Tensor,
topk_weights: Optional[torch.Tensor],
hidden_states: torch.Tensor,
gemm1_weights: torch.Tensor,
output1_scales_scalar: torch.Tensor,
output1_scales_gate_scalar: torch.Tensor,
gemm2_weights: torch.Tensor,
output2_scales_scalar: torch.Tensor,
output: torch.Tensor,
num_experts: int,
top_k: int,
intermediate_size: int,
local_expert_offset: int,
local_num_experts: int,
routed_scaling_factor: Optional[float],
routing_method_type: int,
enable_pdl: Optional[bool] = None,
tune_max_num_tokens: int = 8192,
activation_type: Optional[int] = None,
) -> None:
try:
from flashinfer.fused_moe import trtllm_fp8_per_tensor_scale_routed_moe
except ImportError as e:
raise ImportError(
"FlashInfer TRT-LLM per-tensor FP8 with materialized routing requires "
"trtllm_fp8_per_tensor_scale_routed_moe; use the pinned CUDA FlashInfer build."
) from e
kwargs = dict(
topk_ids=topk_ids if topk_weights is None else (topk_ids, topk_weights),
routing_bias=None,
hidden_states=hidden_states,
gemm1_weights=gemm1_weights,
output1_scales_scalar=output1_scales_scalar,
output1_scales_gate_scalar=output1_scales_gate_scalar,
gemm2_weights=gemm2_weights,
output2_scales_scalar=output2_scales_scalar,
num_experts=num_experts,
top_k=top_k,
n_group=None,
topk_group=None,
intermediate_size=intermediate_size,
local_expert_offset=local_expert_offset,
local_num_experts=local_num_experts,
routed_scaling_factor=routed_scaling_factor,
use_routing_scales_on_input=False,
routing_method_type=routing_method_type,
output=output,
enable_pdl=enable_pdl,
tune_max_num_tokens=tune_max_num_tokens,
)
if activation_type is not None:
from flashinfer.fused_moe.core import ActivationType
kwargs["activation_type"] = ActivationType(activation_type)
trtllm_fp8_per_tensor_scale_routed_moe(**kwargs)
@@ -604,7 +604,10 @@ class FusedMoE(torch.nn.Module):
_fuses_routed_scaling_factor_in_topk(self.quant_method, self.runner_backend)
)
self.routing_method_type = routing_method_type
self.routing_method_type = self.quant_method.get_moe_routing_method_type(
self, routing_method_type
)
self.moe_runner_config.routing_method_type = self.routing_method_type
# overlap args
self.down_gemm_overlap_args: Optional[DownGemmOverlapArgs] = None
@@ -1564,11 +1567,11 @@ class FusedMoE(torch.nn.Module):
return forward_fuseep(self, hidden_states, topk_output)
if is_in_tc_piecewise_cuda_graph():
if (
TopKOutputChecker.format_is_bypassed(topk_output)
and topk_output.expert_location_dispatch_info is not None
if TopKOutputChecker.format_is_bypassed(topk_output) and (
topk_output.expert_location_dispatch_info is not None
or topk_output.num_token_non_padded is not None
):
# The opaque logits-only op cannot carry placement metadata.
# The opaque logits-only op cannot carry padding or placement metadata.
return self.forward_impl(
hidden_states, topk_output, pre_quant_input=pre_quant_input
)
@@ -275,6 +275,9 @@ class KTEPWrapperMethod(FusedMoEMethodBase):
def get_moe_runner_backend(self, layer):
return self.gpu_method.get_moe_runner_backend(layer)
def get_moe_routing_method_type(self, layer, requested_method):
return self.gpu_method.get_moe_routing_method_type(layer, requested_method)
def submit(
self,
layer: torch.nn.Module,
@@ -344,6 +347,12 @@ class KTEPWrapperMethod(FusedMoEMethodBase):
Combined computation results from CPU and GPU experts
"""
from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput
from sglang.srt.layers.moe.topk import TopKOutputChecker
if TopKOutputChecker.format_is_bypassed(dispatch_output.topk_output):
dispatch_output = dispatch_output._replace(
topk_output=dispatch_output.topk_output.to_standard()
)
x = dispatch_output.hidden_states
topk_output = dispatch_output.topk_output
@@ -26,6 +26,7 @@ from sglang.srt.layers.moe.flashinfer_trtllm_moe import (
trtllm_fp8_block_scale_moe_out_wrapper,
trtllm_fp8_block_scale_routed_moe_out_wrapper,
trtllm_fp8_per_tensor_scale_moe_wrapper,
trtllm_fp8_per_tensor_scale_routed_moe_out_wrapper,
)
from sglang.srt.layers.moe.moe_runner.base import (
MoeQuantInfo,
@@ -103,7 +104,6 @@ def _make_deferred_finalize_output(
result,
*,
top_k: int,
materialized_routing: bool = False,
) -> FlashInferTrtllmDeferredFinalizeOutput:
"""Validate and adapt FlashInfer's ``do_finalize=False`` output ABI."""
gemm2_out, expert_weights, expanded_idx_to_permuted_idx = result[:3]
@@ -178,18 +178,24 @@ def _is_gated(layer: Module) -> bool:
FlashInferRouting = torch.Tensor | tuple[torch.Tensor, torch.Tensor]
def _get_routing_for_flashinfer_routed(topk_output) -> FlashInferRouting:
"""Return the `topk_ids` kernel argument for the trtllm routed MoEs."""
def _get_routing_for_flashinfer_routed(
topk_output, *, promote_half_scores: bool = False
) -> FlashInferRouting:
"""Return routed IDs/scores, optionally widening FP16 for native FP8 kernels."""
from sglang.srt.layers.moe.topk import TopKOutputChecker
if TopKOutputChecker.format_is_packed(topk_output):
return topk_output.packed_topk_ids
assert TopKOutputChecker.format_is_standard(topk_output)
return (
topk_output.topk_ids.contiguous(),
topk_output.topk_weights.contiguous(),
)
weights = topk_output.topk_weights
if promote_half_scores and weights.dtype == torch.float16:
# FP8 providers accept BF16/FP32 scores. Widen and make contiguous in
# one copy, retaining the values already rounded by the router.
weights = weights.to(dtype=torch.float32, memory_format=torch.contiguous_format)
else:
weights = weights.contiguous()
return topk_output.topk_ids.contiguous(), weights
def _routing_top_k(routing: FlashInferRouting) -> int:
@@ -771,6 +777,85 @@ class FlashInferTrtllmFp8MoeQuantInfo(MoeQuantInfo):
activation_type: int | None = None
def _run_fp8_per_tensor_scale_moe(
hidden_states: torch.Tensor,
topk_output,
quant_info: FlashInferTrtllmFp8MoeQuantInfo,
runner_config: MoeRunnerConfig,
use_routed_topk: bool,
) -> torch.Tensor:
from sglang.srt.layers.moe.topk import TopKOutputChecker
from sglang.srt.layers.moe.utils import RoutingMethodType
assert quant_info.w13_input_scale is not None
assert quant_info.output1_scales_scalar is not None
assert quant_info.output1_scales_gate_scalar is not None
assert quant_info.output2_scales_scalar is not None
a_q, _ = scaled_fp8_quant(hidden_states, quant_info.w13_input_scale)
with use_symmetric_memory(
get_parallel().tp_group, disabled=not is_allocation_symmetric()
):
output = torch.empty(
hidden_states.shape[0],
hidden_states.shape[1],
dtype=torch.bfloat16,
device=hidden_states.device,
)
common = dict(
hidden_states=a_q,
gemm1_weights=quant_info.w13_weight,
output1_scales_scalar=quant_info.output1_scales_scalar,
output1_scales_gate_scalar=quant_info.output1_scales_gate_scalar,
gemm2_weights=quant_info.w2_weight,
output2_scales_scalar=quant_info.output2_scales_scalar,
num_experts=quant_info.global_num_experts,
intermediate_size=int(quant_info.w2_weight.shape[2]),
local_expert_offset=quant_info.local_expert_offset,
local_num_experts=quant_info.local_num_experts,
routed_scaling_factor=(
runner_config.routed_scaling_factor
if runner_config.routed_scaling_factor is not None
else 1.0
),
tune_max_num_tokens=next_power_of_2(a_q.shape[0]),
activation_type=quant_info.activation_type,
)
if use_routed_topk:
routing = _get_routing_for_flashinfer_routed(
topk_output, promote_half_scores=True
)
ids, weights = routing if isinstance(routing, tuple) else (routing, None)
trtllm_fp8_per_tensor_scale_routed_moe_out_wrapper(
topk_ids=ids,
topk_weights=weights,
output=output,
top_k=_routing_top_k(routing),
# Llama4's native FP8 body also consumes route scores as token scales.
routing_method_type=(
RoutingMethodType.Llama4
if quant_info.routing_method_type == RoutingMethodType.Llama4
else RoutingMethodType.TopK
),
**common,
)
else:
assert TopKOutputChecker.format_is_bypassed(topk_output)
config = topk_output.topk_config
result = trtllm_fp8_per_tensor_scale_moe_wrapper(
routing_logits=topk_output.router_logits.to(torch.bfloat16),
routing_bias=config.correction_bias,
top_k=config.top_k,
n_group=config.num_expert_group,
topk_group=config.topk_group,
use_routing_scales_on_input=False,
routing_method_type=quant_info.routing_method_type,
**common,
)
output.copy_(result)
return output
def fused_experts_none_to_flashinfer_trtllm_fp8(
dispatch_output: StandardDispatchOutput | FlashinferDispatchOutput,
quant_info: FlashInferTrtllmFp8MoeQuantInfo,
@@ -800,6 +885,11 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
else hidden_states.dtype
)
topk_output = dispatch_output.topk_output
if TopKOutputChecker.format_is_bypassed(topk_output) and (
topk_output.num_token_non_padded is not None
or topk_output.expert_location_dispatch_info is not None
):
topk_output = topk_output.to_standard()
materialized_topk = not use_routed_topk and TopKOutputChecker.format_is_standard(
topk_output
)
@@ -823,15 +913,8 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
)
use_shuffled_weight = quant_info.use_mxfp8
defer_finalize = _deferred_finalize_enabled.get()
if defer_finalize and (
not quant_info.block_quant
or use_routed_topk
or not TopKOutputChecker.format_is_bypassed(topk_output)
):
raise RuntimeError(
"FP8 deferred finalize requires block quantization, the logits-based "
"FlashInfer TRTLLM backend, and bypassed TopK"
)
if defer_finalize and not quant_info.block_quant:
raise RuntimeError("FP8 deferred finalize requires block quantization")
if quant_info.block_quant:
assert quant_info.weight_block_k is not None
@@ -884,17 +967,15 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
# during torch.compile for piecewise cuda graph.
# Use custom op wrapper for torch.compile compatibility.
if use_routed_topk:
assert runner_config.top_k is not None, (
"runner_config.top_k is required for flashinfer_trtllm_routed."
routing = _get_routing_for_flashinfer_routed(
topk_output, promote_half_scores=True
)
routing = _get_routing_for_flashinfer_routed(topk_output)
routed_top_k = _routing_top_k(routing)
topk_ids, topk_weights = (
routing if isinstance(routing, tuple) else (routing, None)
)
trtllm_fp8_block_scale_routed_moe_out_wrapper(
topk_ids=topk_ids,
topk_weights=topk_weights,
routed_kwargs = dict(
routing_bias=None,
hidden_states=a_q,
hidden_states_scale=a_sf_t,
@@ -906,7 +987,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
gemm2_weights=quant_info.w2_weight,
gemm2_weights_scale=quant_info.w2_weight_scale_inv,
num_experts=quant_info.global_num_experts,
top_k=runner_config.top_k,
top_k=routed_top_k,
n_group=None,
topk_group=None,
intermediate_size=quant_info.intermediate_size,
@@ -923,12 +1004,38 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
else routing_method_type
),
use_shuffled_weight=use_shuffled_weight,
output=symm_output,
tune_max_num_tokens=next_power_of_2(a_q.shape[0]),
fp8_quantization_type=int(fp8_quantization_type),
activation_type=quant_info.activation_type,
)
output = cast(torch.Tensor, symm_output)
if defer_finalize:
from flashinfer.fused_moe import trtllm_fp8_block_scale_routed_moe
deferred_kwargs = dict(
**routed_kwargs,
topk_ids=routing,
do_finalize=False,
fp8_quantization_type=fp8_quantization_type,
enable_pdl=a_q.shape[0] <= _TRTLLM_MOE_PDL_MAX_TOKENS,
)
if quant_info.activation_type is not None:
from flashinfer.fused_moe.core import ActivationType
deferred_kwargs["activation_type"] = ActivationType(
quant_info.activation_type
)
output = _make_deferred_finalize_output(
trtllm_fp8_block_scale_routed_moe(**deferred_kwargs),
top_k=routed_top_k,
)
else:
trtllm_fp8_block_scale_routed_moe_out_wrapper(
**routed_kwargs,
topk_ids=topk_ids,
topk_weights=topk_weights,
output=symm_output,
fp8_quantization_type=int(fp8_quantization_type),
activation_type=quant_info.activation_type,
)
output = cast(torch.Tensor, symm_output)
else:
assert TopKOutputChecker.format_is_bypassed(topk_output)
common_kwargs = dict(
@@ -987,59 +1094,9 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
)
output = cast(torch.Tensor, symm_output)
else:
assert TopKOutputChecker.format_is_bypassed(topk_output)
assert quant_info.w13_input_scale is not None
assert quant_info.output1_scales_scalar is not None
assert quant_info.output1_scales_gate_scalar is not None
assert quant_info.output2_scales_scalar is not None
a_q, _ = scaled_fp8_quant(hidden_states, quant_info.w13_input_scale)
# Allocate output inside symmetric memory context
with use_symmetric_memory(
get_parallel().tp_group, disabled=not is_allocation_symmetric()
):
symm_output = torch.empty(
hidden_states.shape[0],
hidden_states.shape[1],
dtype=torch.bfloat16,
device=hidden_states.device,
)
# Move kernel call outside context manager to avoid graph breaks
# during torch.compile for piecewise cuda graph.
# Use custom op wrapper for torch.compile compatibility.
router_logits = router_logits.to(torch.bfloat16)
output = trtllm_fp8_per_tensor_scale_moe_wrapper(
routing_logits=router_logits,
routing_bias=correction_bias,
hidden_states=a_q,
gemm1_weights=quant_info.w13_weight,
output1_scales_scalar=quant_info.output1_scales_scalar,
output1_scales_gate_scalar=quant_info.output1_scales_gate_scalar,
gemm2_weights=quant_info.w2_weight,
output2_scales_scalar=quant_info.output2_scales_scalar,
num_experts=quant_info.global_num_experts,
top_k=topk_config.top_k,
n_group=topk_config.num_expert_group,
topk_group=topk_config.topk_group,
intermediate_size=int(quant_info.w2_weight.shape[2]),
local_expert_offset=quant_info.local_expert_offset,
local_num_experts=quant_info.local_num_experts,
routed_scaling_factor=(
runner_config.routed_scaling_factor
if runner_config.routed_scaling_factor is not None
else 1.0
),
use_routing_scales_on_input=False,
routing_method_type=routing_method_type,
tune_max_num_tokens=next_power_of_2(a_q.shape[0]),
activation_type=quant_info.activation_type,
output = _run_fp8_per_tensor_scale_moe(
hidden_states, topk_output, quant_info, runner_config, use_routed_topk
)
symm_output.copy_(output)
output = symm_output
return StandardCombineInput(hidden_states=output)
@@ -1534,7 +1591,6 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
result = _make_deferred_finalize_output(
result,
top_k=_routing_top_k(routing),
materialized_routing=isinstance(routing, tuple),
)
else:
result = result[0]
@@ -1724,7 +1780,6 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
final_hidden_states = _make_deferred_finalize_output(
final_hidden_states,
top_k=runner_config.top_k,
materialized_routing=isinstance(routing, tuple),
)
else:
assert TopKOutputChecker.format_is_bypassed(topk_output)
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
from sglang.srt.layers.moe.moe_runner.base import MoeQuantInfo
from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo
from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput
from sglang.srt.layers.moe.utils import MoeRunnerBackendLike
from sglang.srt.layers.moe.utils import MoeRunnerBackendLike, RoutingMethodType
from sglang.srt.models.utils import WeightsMapper
@@ -123,6 +123,12 @@ class FusedMoEMethodBase(QuantizeMethodBase):
"""
return self.runner.runner_backend if self.runner is not None else None
def get_moe_routing_method_type(
self, layer: torch.nn.Module, requested_method: RoutingMethodType | None
) -> RoutingMethodType | None:
"""Return the routing policy consumed by the selected execution path."""
return requested_method
@abstractmethod
def apply(
self,
+13 -5
View File
@@ -2978,6 +2978,18 @@ class Fp8MoEMethod(FusedMoEMethodBase):
return self.runner.runner_backend
return self._moe_runner_backend
def get_moe_routing_method_type(self, layer, requested_method):
backend = self.get_moe_runner_backend(layer)
if (
requested_method is None
and backend is not None
and (
backend.is_flashinfer_trtllm() or backend.is_flashinfer_trtllm_routed()
)
):
return RoutingMethodType.DeepSeekV3
return requested_method
def get_triton_quant_info(self, layer: torch.nn.Module) -> TritonMoeQuantInfo:
use_rocm_mxfp8 = self.use_mxfp8 and _is_hip and _is_gfx95_supported
return TritonMoeQuantInfo(
@@ -3270,10 +3282,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
is_gated=self.moe_runner_config.is_gated,
)
routing_method_type = layer.routing_method_type
if routing_method_type is None:
routing_method_type = RoutingMethodType.DeepSeekV3
quant_info = FlashInferTrtllmFp8MoeQuantInfo(
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
@@ -3281,7 +3289,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
local_expert_offset=moe_ep_rank * num_local_experts,
local_num_experts=num_local_experts,
intermediate_size=layer.w2_weight.shape[2],
routing_method_type=int(routing_method_type),
routing_method_type=int(layer.routing_method_type),
block_quant=self.block_quant,
use_mxfp8=getattr(self.quant_config, "use_mxfp8", False),
weight_block_k=(
@@ -24,6 +24,7 @@ logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput
from sglang.srt.layers.moe.utils import RoutingMethodType
# MXFP4 group/block size (E8M0 scale per 32 fp4 weights).
_GROUP_SIZE = 32
@@ -127,6 +128,11 @@ class Mxfp4FlashinferCutlassMoEMethod:
def get_moe_runner_backend(self, layer):
return self.runner.runner_backend
def get_moe_routing_method_type(
self, layer: Module, requested_method: RoutingMethodType | None
) -> RoutingMethodType | None:
return requested_method
def process_weights_after_loading(self, layer: Module) -> None:
# Preserve the base FP4 post-load handling.
self._fp8.process_weights_after_loading(layer)
@@ -168,6 +168,11 @@ class Mxfp4FlashinferTrtllmMoEMethod:
def get_moe_runner_backend(self, layer):
return MoeRunnerBackend.FLASHINFER_MXFP4
def get_moe_routing_method_type(
self, layer: Module, requested_method: RoutingMethodType | None
) -> RoutingMethodType | None:
return requested_method
def create_weights(
self,
layer,
@@ -11,6 +11,7 @@ from sglang.srt.utils import log_info_on_rank0
if TYPE_CHECKING:
from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput
from sglang.srt.layers.moe.utils import RoutingMethodType
logger = logging.getLogger(__name__)
@@ -38,6 +39,11 @@ class Mxfp4HummingMoEMethod:
def get_moe_runner_backend(self, layer):
return self.runner.runner_backend
def get_moe_routing_method_type(
self, layer: Module, requested_method: RoutingMethodType | None
) -> RoutingMethodType | None:
return requested_method
def create_weights(
self,
layer: Module,
@@ -13,6 +13,7 @@ from sglang.srt.utils import log_info_on_rank0, round_up, set_weight_attrs
if TYPE_CHECKING:
from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput
from sglang.srt.layers.moe.utils import RoutingMethodType
logger = logging.getLogger(__name__)
@@ -59,6 +60,11 @@ class Mxfp4MarlinMoEMethod:
def get_moe_runner_backend(self, layer):
return self.runner.runner_backend
def get_moe_routing_method_type(
self, layer: Module, requested_method: RoutingMethodType | None
) -> RoutingMethodType | None:
return requested_method
def create_weights(
self,
layer: Module,
@@ -312,7 +312,10 @@ class BailingMoE(nn.Module):
self.topk.topk_config.apply_routed_scaling_factor_on_output = (
self.experts.should_fuse_routed_scaling_factor_in_topk
)
self.topk.bind_runner_backend(self.experts.runner_backend)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
if self.num_shared_experts > 0:
intermediate_size = self.intermediate_size * self.num_shared_experts
+4 -1
View File
@@ -597,7 +597,10 @@ class BailingMoE(nn.Module):
and getattr(quant_config, "is_fp4_experts", False)
),
)
self.topk.bind_runner_backend(self.experts.runner_backend)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
# Whether to apply routed_scaling_factor at model layer.
# For A2A MoE paths (e.g., DeepEP), the runner/post_permute does not apply it,
+4 -1
View File
@@ -682,7 +682,10 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
self.topk.topk_config.apply_routed_scaling_factor_on_output = (
_is_npu or self.experts.should_fuse_routed_scaling_factor_in_topk
)
self.topk.bind_runner_backend(self.experts.runner_backend)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
if self.use_block_routing:
_require_block_routing_runner_compatibility(self.experts, self.top_k)
# shared expert