[Fix] moe: Preserve CUTLASS routing metadata and execution budgets (#45917)
This commit is contained in:
@@ -0,0 +1,194 @@
|
||||
#pragma once
|
||||
|
||||
#include <sgl_kernel/tensor.h>
|
||||
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
#include <tvm/ffi/optional.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
|
||||
namespace sglang {
|
||||
namespace cutlass_moe {
|
||||
|
||||
template <bool kTransposeProblemSizes>
|
||||
__global__ void count_expert_routes(
|
||||
const int32_t* __restrict__ topk_ids,
|
||||
int32_t* __restrict__ problem_sizes1,
|
||||
int32_t* __restrict__ problem_sizes2,
|
||||
int32_t num_routes,
|
||||
int32_t n,
|
||||
int32_t k) {
|
||||
__shared__ int32_t count;
|
||||
if (threadIdx.x == 0) count = 0;
|
||||
__syncthreads();
|
||||
|
||||
const int32_t expert = blockIdx.x;
|
||||
int32_t local_count = 0;
|
||||
for (uint32_t i = threadIdx.x; i < static_cast<uint32_t>(num_routes); i += blockDim.x) {
|
||||
local_count += topk_ids[i] == expert;
|
||||
}
|
||||
atomicAdd(&count, local_count);
|
||||
__syncthreads();
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
constexpr int kCountColumn = kTransposeProblemSizes ? 1 : 0;
|
||||
constexpr int kOutputColumn = 1 - kCountColumn;
|
||||
problem_sizes1[expert * 3 + kCountColumn] = count;
|
||||
problem_sizes1[expert * 3 + kOutputColumn] = 2 * n;
|
||||
problem_sizes1[expert * 3 + 2] = k;
|
||||
problem_sizes2[expert * 3 + kCountColumn] = count;
|
||||
problem_sizes2[expert * 3 + kOutputColumn] = k;
|
||||
problem_sizes2[expert * 3 + 2] = n;
|
||||
}
|
||||
}
|
||||
|
||||
template <bool kTransposeProblemSizes>
|
||||
__global__ void compute_expert_offsets(
|
||||
const int32_t* __restrict__ problem_sizes1,
|
||||
int32_t* __restrict__ expert_offsets,
|
||||
int32_t* __restrict__ blockscale_offsets,
|
||||
int32_t num_experts) {
|
||||
int32_t offset = 0;
|
||||
int32_t blockscale_offset = 0;
|
||||
expert_offsets[0] = 0;
|
||||
if (blockscale_offsets != nullptr) blockscale_offsets[0] = 0;
|
||||
for (int32_t expert = 0; expert < num_experts; ++expert) {
|
||||
constexpr int kCountColumn = kTransposeProblemSizes ? 1 : 0;
|
||||
const int32_t count = problem_sizes1[expert * 3 + kCountColumn];
|
||||
offset += count;
|
||||
expert_offsets[expert + 1] = offset;
|
||||
if (blockscale_offsets != nullptr) {
|
||||
blockscale_offset += (count + 127) / 128 * 128;
|
||||
blockscale_offsets[expert + 1] = blockscale_offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__global__ void complete_route_maps(
|
||||
const int32_t* __restrict__ topk_ids,
|
||||
const int32_t* __restrict__ expert_offsets,
|
||||
int32_t* __restrict__ input_permutation,
|
||||
int32_t* __restrict__ output_permutation,
|
||||
int32_t num_routes,
|
||||
int32_t topk,
|
||||
int32_t num_experts) {
|
||||
const int32_t expert = blockIdx.x;
|
||||
// Exactly one CTA owns each expert's interval, so its cursor is block-local.
|
||||
__shared__ int32_t cursor;
|
||||
if (threadIdx.x == 0) cursor = expert_offsets[expert];
|
||||
__syncthreads();
|
||||
for (uint32_t i = threadIdx.x; i < static_cast<uint32_t>(num_routes); i += blockDim.x) {
|
||||
const int32_t route = topk_ids[i];
|
||||
if (route == expert) {
|
||||
const int32_t destination = atomicAdd(&cursor, 1);
|
||||
input_permutation[destination] = i / topk;
|
||||
output_permutation[i] = destination;
|
||||
} else if (expert == 0 && (route < 0 || route >= num_experts)) {
|
||||
output_permutation[i] = -1;
|
||||
}
|
||||
}
|
||||
|
||||
// The grouped GEMMs use only [0, live); full-capacity gathers also read the tail.
|
||||
const int32_t live = expert_offsets[num_experts];
|
||||
const int64_t thread = static_cast<int64_t>(expert) * blockDim.x + threadIdx.x;
|
||||
const int64_t stride = static_cast<int64_t>(num_experts) * blockDim.x;
|
||||
for (int64_t i = live + thread; i < num_routes; i += stride) {
|
||||
input_permutation[i] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace cutlass_moe
|
||||
|
||||
/// \brief Prepare CUTLASS grouped-GEMM shapes and complete route maps.
|
||||
/// \param input_permutation Destination-to-token map, with zero-filled unused capacity.
|
||||
/// \param output_permutation Route-to-destination map, with -1 for ids outside [0, num_experts).
|
||||
/// \param blockscale_offsets Optional MXFP8 row offsets, rounded up to 128 per expert.
|
||||
template <bool kTransposeProblemSizes = false>
|
||||
inline void cutlass_prepare_moe_input(
|
||||
tvm::ffi::TensorView topk_ids,
|
||||
tvm::ffi::TensorView expert_offsets,
|
||||
tvm::ffi::TensorView problem_sizes1,
|
||||
tvm::ffi::TensorView problem_sizes2,
|
||||
tvm::ffi::TensorView input_permutation,
|
||||
tvm::ffi::TensorView output_permutation,
|
||||
int64_t num_experts,
|
||||
int64_t n,
|
||||
int64_t k,
|
||||
tvm::ffi::Optional<tvm::ffi::TensorView> blockscale_offsets) {
|
||||
using namespace host;
|
||||
constexpr int64_t max_index = std::numeric_limits<int32_t>::max();
|
||||
CHECK_HOST(num_experts > 0 && num_experts <= max_index / 3);
|
||||
CHECK_HOST(n > 0 && n <= max_index / 2 && k > 0 && k <= max_index);
|
||||
|
||||
auto device = SymbolicDevice{};
|
||||
auto tokens = SymbolicSize{"tokens"};
|
||||
auto topk = SymbolicSize{"topk"};
|
||||
TensorMatcher({tokens, topk})
|
||||
.with_strides({-1, -1})
|
||||
.with_dtype<int32_t>()
|
||||
.with_device<kDLCUDA>(device)
|
||||
.verify(topk_ids);
|
||||
const int64_t num_routes = topk_ids.numel();
|
||||
// Empty PyTorch dimensions can have noncanonical strides; no ID is read.
|
||||
RuntimeCheck(num_routes == 0 || topk_ids.is_contiguous(), "topk_ids must be contiguous when nonempty");
|
||||
CHECK_HOST(num_routes <= max_index);
|
||||
TensorMatcher({num_experts + 1}).with_dtype<int32_t>().with_device(device).verify(expert_offsets);
|
||||
TensorMatcher({num_experts, 3})
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device)
|
||||
.verify(problem_sizes1)
|
||||
.verify(problem_sizes2);
|
||||
TensorMatcher({num_routes})
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device)
|
||||
.verify(input_permutation)
|
||||
.verify(output_permutation);
|
||||
|
||||
int32_t* blockscale_ptr = nullptr;
|
||||
if (blockscale_offsets.has_value()) {
|
||||
TensorMatcher({num_experts + 1}).with_dtype<int32_t>().with_device(device).verify(blockscale_offsets.value());
|
||||
CHECK_HOST(num_routes + 127 * std::min(num_experts, num_routes) <= max_index);
|
||||
blockscale_ptr = static_cast<int32_t*>(blockscale_offsets.value().data_ptr());
|
||||
}
|
||||
|
||||
const auto threads = static_cast<uint32_t>(std::min<int64_t>(512, std::max<int64_t>(1, num_routes)));
|
||||
const auto experts = static_cast<uint32_t>(num_experts);
|
||||
const auto routes = static_cast<int32_t>(num_routes);
|
||||
const auto* ids = static_cast<const int32_t*>(topk_ids.data_ptr());
|
||||
auto* sizes1 = static_cast<int32_t*>(problem_sizes1.data_ptr());
|
||||
auto* sizes2 = static_cast<int32_t*>(problem_sizes2.data_ptr());
|
||||
auto* offsets = static_cast<int32_t*>(expert_offsets.data_ptr());
|
||||
const auto stream = LaunchKernel::resolve_device(device.unwrap());
|
||||
|
||||
LaunchKernel(dim3(experts), dim3(threads), stream)(
|
||||
cutlass_moe::count_expert_routes<kTransposeProblemSizes>,
|
||||
ids,
|
||||
sizes1,
|
||||
sizes2,
|
||||
routes,
|
||||
static_cast<int32_t>(n),
|
||||
static_cast<int32_t>(k));
|
||||
LaunchKernel(dim3(1), dim3(1), stream)(
|
||||
cutlass_moe::compute_expert_offsets<kTransposeProblemSizes>,
|
||||
sizes1,
|
||||
offsets,
|
||||
blockscale_ptr,
|
||||
static_cast<int32_t>(num_experts));
|
||||
if (num_routes != 0) {
|
||||
LaunchKernel(dim3(experts), dim3(threads), stream)(
|
||||
cutlass_moe::complete_route_maps,
|
||||
ids,
|
||||
offsets,
|
||||
static_cast<int32_t*>(input_permutation.data_ptr()),
|
||||
static_cast<int32_t*>(output_permutation.data_ptr()),
|
||||
routes,
|
||||
static_cast<int32_t>(topk.unwrap()),
|
||||
static_cast<int32_t>(num_experts));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace sglang
|
||||
@@ -0,0 +1,142 @@
|
||||
#pragma once
|
||||
|
||||
#include <sgl_kernel/tensor.h>
|
||||
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <type_traits>
|
||||
|
||||
namespace sglang {
|
||||
namespace shuffle_weighted_sum {
|
||||
|
||||
SGL_DEVICE float accumulate(float value, float factor, float total) {
|
||||
float result;
|
||||
// Keep the slot-order FMA, including +0 * +0 for inactive slots. No FTZ.
|
||||
asm volatile("fma.rn.f32 %0, %1, %2, %3;" : "=f"(result) : "f"(value), "f"(factor), "f"(total));
|
||||
return result;
|
||||
}
|
||||
|
||||
template <
|
||||
typename scalar_t,
|
||||
typename weight_t,
|
||||
typename index_t,
|
||||
int64_t kTopK,
|
||||
int kThreads,
|
||||
bool kCastWeightsToInputDType>
|
||||
__global__ void reduce(
|
||||
const scalar_t* inputs,
|
||||
scalar_t* output,
|
||||
const index_t* permutation,
|
||||
const weight_t* weights,
|
||||
int64_t width,
|
||||
int64_t rows,
|
||||
int64_t input_stride,
|
||||
int64_t output_stride,
|
||||
int64_t weight_stride0,
|
||||
int64_t weight_stride1) {
|
||||
constexpr int kItems = 2;
|
||||
const int64_t token = blockIdx.x;
|
||||
const int64_t first_column = static_cast<int64_t>(blockIdx.y) * kThreads * kItems + threadIdx.x;
|
||||
if (first_column >= width) return;
|
||||
float totals[kItems] = {0.0f, 0.0f};
|
||||
for (int64_t slot = 0; slot < kTopK; ++slot) {
|
||||
const int64_t row = permutation[token * kTopK + slot];
|
||||
const bool active = row >= 0 && row < rows;
|
||||
float factor = 0.0f;
|
||||
if (active) {
|
||||
factor = device::cast<float>(weights[token * weight_stride0 + slot * weight_stride1]);
|
||||
if constexpr (kCastWeightsToInputDType && !std::is_same_v<scalar_t, weight_t>) {
|
||||
// Match scores.to(inputs.dtype) before the FP32 slot-order FMA.
|
||||
factor = device::cast<float>(device::cast<scalar_t>(factor));
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int item = 0; item < kItems; ++item) {
|
||||
const int64_t column = first_column + item * kThreads;
|
||||
float value = 0.0f;
|
||||
if (active && column < width) value = device::cast<float>(inputs[row * input_stride + column]);
|
||||
totals[item] = accumulate(value, factor, totals[item]);
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int item = 0; item < kItems; ++item) {
|
||||
const int64_t column = first_column + item * kThreads;
|
||||
if (column < width) output[token * output_stride + column] = device::cast<scalar_t>(totals[item]);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace shuffle_weighted_sum
|
||||
|
||||
/// \brief Reduce mapped rows in slot order without reading inactive payloads/scores.
|
||||
/// \param permutation Contiguous int32/int64 map of any rank, one entry per weight.
|
||||
/// \param weights Potentially strided scores; rows outside the input extent are inactive.
|
||||
template <
|
||||
typename scalar_t,
|
||||
typename weight_t,
|
||||
typename index_t,
|
||||
int64_t kTopK,
|
||||
int kThreads,
|
||||
bool kCastWeightsToInputDType>
|
||||
void shuffle_weighted_sum_cuda(
|
||||
tvm::ffi::TensorView inputs,
|
||||
tvm::ffi::TensorView output,
|
||||
tvm::ffi::TensorView permutation,
|
||||
tvm::ffi::TensorView weights) {
|
||||
using namespace host;
|
||||
static_assert(std::is_same_v<scalar_t, fp16_t> || std::is_same_v<scalar_t, bf16_t>);
|
||||
static_assert(
|
||||
std::is_same_v<weight_t, fp16_t> || std::is_same_v<weight_t, bf16_t> || std::is_same_v<weight_t, float>);
|
||||
static_assert(std::is_same_v<index_t, int32_t> || std::is_same_v<index_t, int64_t>);
|
||||
static_assert(kTopK >= 0 && kThreads > 0 && kThreads <= 1024);
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
auto rows = SymbolicSize{"input_rows"};
|
||||
auto tokens = SymbolicSize{"output_rows"};
|
||||
auto width = SymbolicSize{"width"};
|
||||
auto input_stride = SymbolicSize{"input_stride"};
|
||||
auto output_stride = SymbolicSize{"output_stride"};
|
||||
auto weight_stride0 = SymbolicSize{"weight_stride0"};
|
||||
auto weight_stride1 = SymbolicSize{"weight_stride1"};
|
||||
TensorMatcher({rows, width})
|
||||
.with_strides({input_stride, 1})
|
||||
.with_dtype<scalar_t>()
|
||||
.with_device(device)
|
||||
.verify(inputs);
|
||||
TensorMatcher({tokens, width})
|
||||
.with_strides({output_stride, 1})
|
||||
.with_dtype<scalar_t>()
|
||||
.with_device(device)
|
||||
.verify(output);
|
||||
TensorMatcher({tokens, kTopK})
|
||||
.with_strides({weight_stride0, weight_stride1})
|
||||
.with_dtype<weight_t>()
|
||||
.with_device(device)
|
||||
.verify(weights);
|
||||
auto map_dtype = SymbolicDType{};
|
||||
map_dtype.set_options<index_t>();
|
||||
map_dtype.verify(permutation.dtype());
|
||||
device.verify(permutation.device());
|
||||
CHECK_HOST(permutation.is_contiguous() && permutation.numel() == weights.numel());
|
||||
if (tokens.unwrap() == 0 || width.unwrap() == 0) return;
|
||||
constexpr int64_t kTile = kThreads * 2;
|
||||
const int64_t tiles = div_ceil(width.unwrap(), kTile);
|
||||
// These are the same x/y grid limits as the Triton fallback.
|
||||
CHECK_HOST(tokens.unwrap() <= std::numeric_limits<int32_t>::max() && tiles <= 65535);
|
||||
LaunchKernel(dim3(static_cast<uint32_t>(tokens.unwrap()), static_cast<uint32_t>(tiles)), kThreads, device.unwrap())(
|
||||
shuffle_weighted_sum::reduce<scalar_t, weight_t, index_t, kTopK, kThreads, kCastWeightsToInputDType>,
|
||||
static_cast<const scalar_t*>(inputs.data_ptr()),
|
||||
static_cast<scalar_t*>(output.data_ptr()),
|
||||
static_cast<const index_t*>(permutation.data_ptr()),
|
||||
static_cast<const weight_t*>(weights.data_ptr()),
|
||||
width.unwrap(),
|
||||
rows.unwrap(),
|
||||
input_stride.unwrap(),
|
||||
output_stride.unwrap(),
|
||||
weight_stride0.unwrap(),
|
||||
weight_stride1.unwrap());
|
||||
}
|
||||
|
||||
} // namespace sglang
|
||||
@@ -71,10 +71,19 @@ register_kernel(
|
||||
)
|
||||
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.cutlass_prepare_moe_input",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.moe.cutlass_prepare_moe_input:cutlass_prepare_moe_input",
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.shuffle_weighted_sum",
|
||||
backend=KernelBackend.TRITON,
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.moe.shuffle_weighted_sum:shuffle_weighted_sum",
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
@@ -112,6 +121,15 @@ register_kernel(
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.compute_cutlass_w4a8_moe_input_scale",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.moe.ep_moe_kernels:compute_cutlass_w4a8_moe_input_scale",
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def moe_align_block_size(
|
||||
topk_ids: torch.Tensor,
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Complete routing maps and grouped-GEMM shapes for CUTLASS MoE."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_cutlass_prepare_moe_input_module(
|
||||
transpose_problem_sizes: bool = False,
|
||||
) -> Module:
|
||||
args = make_cpp_args(transpose_problem_sizes)
|
||||
return load_jit(
|
||||
"cutlass_prepare_moe_input",
|
||||
*args,
|
||||
cuda_files=["moe/cutlass_prepare_moe_input.cuh"],
|
||||
cuda_wrappers=[("prepare", f"cutlass_prepare_moe_input<{args}>")],
|
||||
)
|
||||
|
||||
|
||||
@register_custom_op(
|
||||
op_name="cutlass_prepare_moe_input",
|
||||
mutates_args=[
|
||||
"expert_offsets",
|
||||
"problem_sizes1",
|
||||
"problem_sizes2",
|
||||
"input_permutation",
|
||||
"output_permutation",
|
||||
"blockscale_offsets",
|
||||
],
|
||||
)
|
||||
def cutlass_prepare_moe_input(
|
||||
topk_ids: torch.Tensor,
|
||||
expert_offsets: torch.Tensor,
|
||||
problem_sizes1: torch.Tensor,
|
||||
problem_sizes2: torch.Tensor,
|
||||
input_permutation: torch.Tensor,
|
||||
output_permutation: torch.Tensor,
|
||||
num_experts: int,
|
||||
n: int,
|
||||
k: int,
|
||||
blockscale_offsets: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
transpose_problem_sizes: bool = False,
|
||||
) -> None:
|
||||
"""Write contiguous int32 CUTLASS metadata, including every unused map entry.
|
||||
|
||||
Routes outside [0, num_experts) have output permutation -1. Input-map
|
||||
capacity beyond expert_offsets[-1] points at token zero for full gathers.
|
||||
Optional blockscale offsets count per-expert rows rounded up to 128.
|
||||
Transposed problem sizes put the token count in N instead of M.
|
||||
"""
|
||||
_jit_cutlass_prepare_moe_input_module(transpose_problem_sizes).prepare(
|
||||
topk_ids,
|
||||
expert_offsets,
|
||||
problem_sizes1,
|
||||
problem_sizes2,
|
||||
input_permutation,
|
||||
output_permutation,
|
||||
num_experts,
|
||||
n,
|
||||
k,
|
||||
blockscale_offsets,
|
||||
)
|
||||
@@ -145,13 +145,27 @@ def deepep_post_reorder_triton_kernel(
|
||||
|
||||
@triton.jit
|
||||
def compute_src2dst_triton_kernel(
|
||||
reorder_ids, src2dst, num_toks, BLOCK_SIZE: tl.constexpr
|
||||
sorted_topk_ids, reorder_ids, src2dst, num_toks, BLOCK_SIZE: tl.constexpr
|
||||
):
|
||||
first_nonnegative = 0
|
||||
if tl.load(sorted_topk_ids) < 0:
|
||||
left, right = 0, num_toks
|
||||
while left < right:
|
||||
middle = left + (right - left) // 2
|
||||
expert_id = tl.load(sorted_topk_ids + middle)
|
||||
if expert_id < 0:
|
||||
left = middle + 1
|
||||
else:
|
||||
right = middle
|
||||
first_nonnegative = left
|
||||
|
||||
pid = tl.program_id(axis=0)
|
||||
dst_id = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = dst_id < num_toks
|
||||
src_id = tl.load(reorder_ids + dst_id, mask=mask)
|
||||
tl.store(src2dst + src_id, dst_id, mask=mask)
|
||||
# Native expert offsets omit negative IDs, so packed rows must omit that prefix.
|
||||
packed_id = tl.where(dst_id >= first_nonnegative, dst_id - first_nonnegative, -1)
|
||||
tl.store(src2dst + src_id, packed_id, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -189,18 +203,95 @@ def deepep_run_moe_deep_preprocess(topk_ids: torch.Tensor, num_experts: int):
|
||||
|
||||
|
||||
def cutlass_w4_run_moe_ep_preproess(topk_ids: torch.Tensor):
|
||||
_, reorder_ids = torch.sort(topk_ids.view(-1), stable=True)
|
||||
src2dst = torch.empty(topk_ids.numel(), device=topk_ids.device, dtype=torch.int32)
|
||||
if topk_ids.numel() == 0:
|
||||
return src2dst
|
||||
sorted_topk_ids, reorder_ids = torch.sort(topk_ids.view(-1), stable=True)
|
||||
|
||||
BLOCK_SIZE = 512
|
||||
grid = (triton.cdiv(topk_ids.numel(), BLOCK_SIZE),)
|
||||
src2dst = torch.empty(topk_ids.numel(), device=topk_ids.device, dtype=torch.int32)
|
||||
compute_src2dst_triton_kernel[grid](
|
||||
reorder_ids, src2dst, topk_ids.numel(), BLOCK_SIZE
|
||||
sorted_topk_ids, reorder_ids, src2dst, topk_ids.numel(), BLOCK_SIZE
|
||||
)
|
||||
|
||||
return src2dst
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _compute_cutlass_w4a8_moe_input_scale_kernel(
|
||||
input_ptr,
|
||||
topk_ids_ptr,
|
||||
output_s_ptr,
|
||||
stride_input_row,
|
||||
stride_input_col,
|
||||
stride_ids_row,
|
||||
stride_ids_col,
|
||||
hidden_size: tl.constexpr,
|
||||
topk: tl.constexpr,
|
||||
num_local_experts: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
routes = tl.arange(0, BLOCK_TOPK)
|
||||
expert_ids = tl.load(
|
||||
topk_ids_ptr + row * stride_ids_row + routes * stride_ids_col,
|
||||
mask=routes < topk,
|
||||
other=-1,
|
||||
)
|
||||
local_route = (expert_ids >= 0) & (expert_ids < num_local_experts)
|
||||
active = tl.sum(local_route.to(tl.int32), 0) > 0
|
||||
columns = tl.arange(0, BLOCK_SIZE)
|
||||
values = tl.load(
|
||||
input_ptr + row * stride_input_row + columns * stride_input_col,
|
||||
mask=(columns < hidden_size) & active,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
absmax = tl.max(tl.abs(values), 0)
|
||||
tl.atomic_max(output_s_ptr, absmax / FP8_MAX, sem="relaxed")
|
||||
|
||||
|
||||
def compute_cutlass_w4a8_moe_input_scale(
|
||||
input: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
output_s: torch.Tensor,
|
||||
num_local_experts: int,
|
||||
):
|
||||
"""Accumulate the FP8 scale from rows with a local expert route.
|
||||
|
||||
The caller initializes output_s to its minimum finite scale. Inputs and IDs
|
||||
are preserved, and fully inactive rows never load activations, including NaNs.
|
||||
"""
|
||||
assert input.ndim == 2
|
||||
assert input.dtype in (torch.float16, torch.bfloat16, torch.float32)
|
||||
assert topk_ids.ndim == 2 and topk_ids.shape[0] == input.shape[0]
|
||||
assert topk_ids.dtype in (torch.int32, torch.int64)
|
||||
assert output_s.numel() == 1 and output_s.dtype == torch.float32
|
||||
assert input.is_cuda
|
||||
assert input.device == topk_ids.device == output_s.device
|
||||
if input.numel() == 0 or topk_ids.shape[1] == 0:
|
||||
return
|
||||
hidden_size = input.shape[1]
|
||||
block_size = triton.next_power_of_2(max(1, hidden_size))
|
||||
_compute_cutlass_w4a8_moe_input_scale_kernel[(input.shape[0],)](
|
||||
input,
|
||||
topk_ids,
|
||||
output_s,
|
||||
input.stride(0),
|
||||
input.stride(1),
|
||||
topk_ids.stride(0),
|
||||
topk_ids.stride(1),
|
||||
hidden_size=hidden_size,
|
||||
topk=topk_ids.shape[1],
|
||||
num_local_experts=num_local_experts,
|
||||
FP8_MAX=torch.finfo(torch.float8_e4m3fn).max,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_TOPK=triton.next_power_of_2(topk_ids.shape[1]),
|
||||
num_warps=4 if block_size < 4096 else 8,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def pre_reorder_triton_kernel_for_cutlass_moe(
|
||||
input_ptr,
|
||||
@@ -241,7 +332,7 @@ def pre_reorder_triton_kernel_for_cutlass_moe(
|
||||
out_data = (in_data * a1_scale).to(OutDtype)
|
||||
for idx in range(topk):
|
||||
expert_id = tl.load(token_topk_ids_ptr + idx)
|
||||
if expert_id != num_local_experts:
|
||||
if (expert_id >= 0) & (expert_id < num_local_experts):
|
||||
dst_idx = tl.load(token_src2dst_ptr + idx).to(tl.int64)
|
||||
tl.store(dst_ptr_offs + dst_idx * hidden_size, out_data, mask=mask)
|
||||
|
||||
@@ -855,7 +946,7 @@ def post_reorder_triton_kernel_for_cutlass_moe(
|
||||
sum_vec = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
|
||||
for idx in range(topk):
|
||||
expert_id = tl.load(token_topk_ids_ptr + idx)
|
||||
if expert_id != num_local_experts:
|
||||
if (expert_id >= 0) & (expert_id < num_local_experts):
|
||||
dst_idx_int32 = tl.load(token_src2dst_ptr + idx)
|
||||
dst_idx = dst_idx_int32.to(tl.int64)
|
||||
dst_idx = dst_idx
|
||||
|
||||
@@ -1,9 +1,56 @@
|
||||
"""Reduce routed rows while skipping inactive permutation entries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
is_hip_runtime,
|
||||
is_musa_runtime,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
from sglang.kernels.jit.utils.common import is_sail_runtime
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_shuffle_weighted_sum_module(
|
||||
dtype: torch.dtype,
|
||||
output_dtype: torch.dtype,
|
||||
weight_dtype: torch.dtype,
|
||||
index_dtype: torch.dtype,
|
||||
topk: int,
|
||||
cast_weights_to_input_dtype: bool = False,
|
||||
) -> Module | None:
|
||||
if (
|
||||
dtype not in (torch.float16, torch.bfloat16)
|
||||
or output_dtype != dtype
|
||||
or weight_dtype not in (torch.float16, torch.bfloat16, torch.float32)
|
||||
or index_dtype not in (torch.int32, torch.int64)
|
||||
or is_hip_runtime()
|
||||
or is_musa_runtime()
|
||||
or is_sail_runtime()
|
||||
):
|
||||
return None
|
||||
args = make_cpp_args(
|
||||
dtype, weight_dtype, index_dtype, topk, 128, cast_weights_to_input_dtype
|
||||
)
|
||||
return load_jit(
|
||||
"shuffle_weighted_sum",
|
||||
*args,
|
||||
cuda_files=["moe/shuffle_weighted_sum.cuh"],
|
||||
cuda_wrappers=[("reduce", f"shuffle_weighted_sum_cuda<{args}>")],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["rows", "weight_stride0", "weight_stride1"])
|
||||
def _shuffle_weighted_sum_kernel(
|
||||
@@ -19,6 +66,7 @@ def _shuffle_weighted_sum_kernel(
|
||||
weight_stride0,
|
||||
weight_stride1,
|
||||
BLOCK: tl.constexpr,
|
||||
CAST_WEIGHTS_TO_INPUT_DTYPE: tl.constexpr = False,
|
||||
):
|
||||
token = tl.program_id(0)
|
||||
columns = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
@@ -35,7 +83,10 @@ def _shuffle_weighted_sum_kernel(
|
||||
weights + token * weight_stride0 + slot * weight_stride1,
|
||||
mask=active,
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
)
|
||||
if CAST_WEIGHTS_TO_INPUT_DTYPE:
|
||||
factor = factor.to(inputs.dtype.element_ty)
|
||||
factor = factor.to(tl.float32)
|
||||
total += values * factor
|
||||
tl.store(
|
||||
output + token.to(tl.int64) * output_stride + columns,
|
||||
@@ -44,13 +95,15 @@ def _shuffle_weighted_sum_kernel(
|
||||
)
|
||||
|
||||
|
||||
def shuffle_weighted_sum(
|
||||
def _shuffle_weighted_sum_triton(
|
||||
inputs: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
permutation: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
*,
|
||||
cast_weights_to_input_dtype: bool = False,
|
||||
) -> None:
|
||||
"""Sum mapped expert outputs; inactive maps read neither data nor weights."""
|
||||
"""Sum mapped rows, optionally rounding scores to the input dtype first."""
|
||||
assert inputs.ndim == output.ndim == weights.ndim == 2
|
||||
assert inputs.shape[1] == output.shape[1]
|
||||
assert inputs.stride(1) == output.stride(1) == 1
|
||||
@@ -58,6 +111,17 @@ def shuffle_weighted_sum(
|
||||
assert permutation.is_contiguous() and permutation.numel() == weights.numel()
|
||||
if output.numel() == 0:
|
||||
return
|
||||
if cast_weights_to_input_dtype and (
|
||||
inputs.dtype not in (torch.float16, torch.bfloat16)
|
||||
or weights.dtype not in (torch.float16, torch.bfloat16, torch.float32)
|
||||
or not inputs.is_cuda
|
||||
or is_hip_runtime()
|
||||
or is_musa_runtime()
|
||||
or is_sail_runtime()
|
||||
):
|
||||
# Preserve Torch conversion semantics outside the fused CUDA types.
|
||||
weights = weights.to(inputs.dtype)
|
||||
cast_weights_to_input_dtype = False
|
||||
_shuffle_weighted_sum_kernel[(output.shape[0], triton.cdiv(output.shape[1], 256))](
|
||||
inputs,
|
||||
output,
|
||||
@@ -71,4 +135,41 @@ def shuffle_weighted_sum(
|
||||
weights.stride(0),
|
||||
weights.stride(1),
|
||||
BLOCK=256,
|
||||
CAST_WEIGHTS_TO_INPUT_DTYPE=cast_weights_to_input_dtype,
|
||||
)
|
||||
|
||||
|
||||
@register_custom_op(op_name="shuffle_weighted_sum", mutates_args=["output"])
|
||||
def shuffle_weighted_sum(
|
||||
inputs: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
permutation: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
*,
|
||||
cast_weights_to_input_dtype: bool = False,
|
||||
) -> None:
|
||||
"""Sum mapped rows, optionally rounding scores to the input dtype first."""
|
||||
if (
|
||||
inputs.is_cuda
|
||||
and inputs.dtype in (torch.float16, torch.bfloat16)
|
||||
and inputs.ndim == output.ndim == weights.ndim == 2
|
||||
and output.numel() != 0
|
||||
):
|
||||
module = _jit_shuffle_weighted_sum_module(
|
||||
inputs.dtype,
|
||||
output.dtype,
|
||||
weights.dtype,
|
||||
permutation.dtype,
|
||||
weights.shape[1],
|
||||
cast_weights_to_input_dtype,
|
||||
)
|
||||
if module is not None:
|
||||
module.reduce(inputs, output, permutation, weights)
|
||||
return
|
||||
_shuffle_weighted_sum_triton(
|
||||
inputs,
|
||||
output,
|
||||
permutation,
|
||||
weights,
|
||||
cast_weights_to_input_dtype=cast_weights_to_input_dtype,
|
||||
)
|
||||
|
||||
@@ -14,11 +14,13 @@ if _is_cuda:
|
||||
es_sm100_mxfp8_blockscaled_grouped_mm,
|
||||
es_sm100_mxfp8_blockscaled_grouped_quant,
|
||||
fp8_blockwise_scaled_grouped_mm,
|
||||
prepare_moe_input,
|
||||
shuffle_rows,
|
||||
)
|
||||
|
||||
from sglang.kernels.ops.activation.activation import silu_and_mul
|
||||
from sglang.kernels.ops.moe.cutlass_prepare_moe_input import (
|
||||
cutlass_prepare_moe_input,
|
||||
)
|
||||
from sglang.kernels.ops.moe.shuffle_rows_with_scales import (
|
||||
shuffle_rows_with_scales,
|
||||
)
|
||||
@@ -142,10 +144,6 @@ def cutlass_fused_experts_fp8(
|
||||
topk = topk_ids.size(1)
|
||||
device = a.device
|
||||
|
||||
# prepare_moe_input writes only live routes; gathers still span the tail.
|
||||
a_map = torch.zeros((topk_ids.numel()), dtype=torch.int32, device=device)
|
||||
c_map = torch.full((topk_ids.numel(),), -1, dtype=torch.int32, device=device)
|
||||
|
||||
if use_mxfp8:
|
||||
assert es_up and es_down, "MXFP8 requires expert-specialization for both GEMMs"
|
||||
assert get_platform().is_sm100, "MXFP8 requires SM100"
|
||||
@@ -178,13 +176,23 @@ def cutlass_fused_experts_fp8(
|
||||
(max_total + mxfp8_blockscale_align - 1) // mxfp8_blockscale_align
|
||||
) * mxfp8_blockscale_align
|
||||
|
||||
if m == 0 or topk == 0:
|
||||
if output is None:
|
||||
output = torch.empty((m, k), dtype=out_dtype, device=device)
|
||||
if m != 0:
|
||||
output.zero_()
|
||||
return output
|
||||
|
||||
a_map = torch.empty((topk_ids.numel(),), dtype=torch.int32, device=device)
|
||||
c_map = torch.empty((topk_ids.numel(),), dtype=torch.int32, device=device)
|
||||
|
||||
blockscale_offsets = None
|
||||
if use_mxfp8 and (es_up or es_down):
|
||||
blockscale_offsets = torch.empty(
|
||||
(num_experts + 1,), dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
prepare_moe_input(
|
||||
cutlass_prepare_moe_input(
|
||||
topk_ids,
|
||||
expert_offsets,
|
||||
problem_sizes1,
|
||||
@@ -346,5 +354,7 @@ def cutlass_fused_experts_fp8(
|
||||
if output is None:
|
||||
output = torch.empty((m, k), device=device, dtype=out_dtype)
|
||||
|
||||
shuffle_weighted_sum(c2, output, c_map, topk_weights.to(out_dtype))
|
||||
shuffle_weighted_sum(
|
||||
c2, output, c_map, topk_weights, cast_weights_to_input_dtype=True
|
||||
)
|
||||
return output
|
||||
|
||||
@@ -18,10 +18,14 @@ if _is_cuda_alike:
|
||||
|
||||
if _is_cuda:
|
||||
from sglang.kernels.ops.activation.activation import silu_and_mul
|
||||
from sglang.kernels.ops.moe.cutlass_prepare_moe_input import (
|
||||
cutlass_prepare_moe_input,
|
||||
)
|
||||
else:
|
||||
from sgl_kernel import silu_and_mul
|
||||
|
||||
from sglang.kernels.ops.moe.ep_moe_kernels import (
|
||||
compute_cutlass_w4a8_moe_input_scale,
|
||||
cutlass_w4_run_moe_ep_preproess,
|
||||
deepep_ll_get_cutlass_w4a8_moe_mm_data,
|
||||
deepep_permute_triton_kernel,
|
||||
@@ -34,10 +38,7 @@ from sglang.kernels.ops.moe.ep_moe_kernels import (
|
||||
silu_mul_dynamic_tensorwise_quant_for_cutlass_moe,
|
||||
silu_mul_static_tensorwise_quant_for_cutlass_moe,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.per_tensor_quant_fp8 import (
|
||||
per_tensor_absmax_fp8,
|
||||
per_tensor_quant_fp8,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.per_tensor_quant_fp8 import per_tensor_quant_fp8
|
||||
|
||||
|
||||
def cutlass_w4a8_moe(
|
||||
@@ -85,6 +86,7 @@ def cutlass_w4a8_moe(
|
||||
Shape: [num_experts, N // 512, K * 4]
|
||||
- topk_weights (torch.Tensor): The weights of each token->expert mapping.
|
||||
- topk_ids (torch.Tensor): The ids of each token->expert mapping.
|
||||
Noncontiguous IDs are materialized once for the native flat-pointer consumers.
|
||||
- a_strides1 (torch.Tensor): The input strides of the first grouped gemm.
|
||||
- b_strides1 (torch.Tensor): The weights strides of the first grouped gemm.
|
||||
- c_strides1 (torch.Tensor): The output strides of the first grouped gemm.
|
||||
@@ -126,13 +128,42 @@ def cutlass_w4a8_moe(
|
||||
if apply_router_weight_on_input:
|
||||
assert topk == 1, "apply_router_weight_on_input is only implemented for topk=1"
|
||||
|
||||
device = a.device
|
||||
# Both graph padding and nonlocal routes use -1, including without EP.
|
||||
topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids)
|
||||
# Native expert counts and reorder kernels read contiguous int32 IDs.
|
||||
topk_ids = topk_ids.contiguous()
|
||||
|
||||
src2dst = cutlass_w4_run_moe_ep_preproess(
|
||||
topk_ids,
|
||||
)
|
||||
device = a.device
|
||||
if a1_scale is None:
|
||||
a1_scale = torch.full(
|
||||
(1,),
|
||||
1e-10 / torch.finfo(torch.float8_e4m3fn).max,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
compute_cutlass_w4a8_moe_input_scale(
|
||||
input=a,
|
||||
topk_ids=topk_ids,
|
||||
output_s=a1_scale,
|
||||
num_local_experts=num_local_experts,
|
||||
)
|
||||
|
||||
if _is_cuda:
|
||||
a_map = torch.empty(topk_ids.numel(), dtype=torch.int32, device=device)
|
||||
c_map = torch.empty(topk_ids.numel(), dtype=torch.int32, device=device)
|
||||
cutlass_prepare_moe_input(
|
||||
topk_ids,
|
||||
expert_offsets,
|
||||
problem_sizes1,
|
||||
problem_sizes2,
|
||||
a_map,
|
||||
c_map,
|
||||
num_local_experts,
|
||||
n,
|
||||
k,
|
||||
transpose_problem_sizes=True,
|
||||
)
|
||||
src2dst = c_map
|
||||
else:
|
||||
src2dst = cutlass_w4_run_moe_ep_preproess(topk_ids)
|
||||
|
||||
gateup_input = torch.empty(
|
||||
(m * topk, k),
|
||||
@@ -140,15 +171,6 @@ def cutlass_w4a8_moe(
|
||||
dtype=torch.float8_e4m3fn,
|
||||
)
|
||||
|
||||
# TODO: fuse per_tensor_absmax_fp8 and pre_reorder_for_cutlass_moe
|
||||
if a1_scale is None:
|
||||
a1_scale = torch.zeros(1, dtype=torch.float32, device=device)
|
||||
# Inactive rows must not change the quantization of live expert inputs.
|
||||
inactive_rows = (topk_ids == num_local_experts).all(dim=-1, keepdim=True)
|
||||
scale_input = a.masked_fill(inactive_rows, 0.0)
|
||||
per_tensor_absmax_fp8(scale_input, a1_scale)
|
||||
a1_scale.clamp_(min=1e-10 / torch.finfo(torch.float8_e4m3fn).max)
|
||||
|
||||
pre_reorder_for_cutlass_moe(
|
||||
a,
|
||||
gateup_input,
|
||||
@@ -161,22 +183,20 @@ def cutlass_w4a8_moe(
|
||||
k,
|
||||
)
|
||||
|
||||
# NOTE: a_map and c_map are not used in the get_cutlass_w4a8_moe_mm_data kernel,
|
||||
# they are kept to allow for a quick switch of the permutation logic
|
||||
# from the current triton kernel implementation to the cutlass-based one if needed.
|
||||
a_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
|
||||
c_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
|
||||
get_cutlass_w4a8_moe_mm_data(
|
||||
topk_ids,
|
||||
expert_offsets,
|
||||
problem_sizes1,
|
||||
problem_sizes2,
|
||||
a_map,
|
||||
c_map,
|
||||
num_local_experts,
|
||||
n,
|
||||
k,
|
||||
)
|
||||
if not _is_cuda:
|
||||
a_map = torch.empty(topk_ids.numel(), dtype=torch.int32, device=device)
|
||||
c_map = torch.empty(topk_ids.numel(), dtype=torch.int32, device=device)
|
||||
get_cutlass_w4a8_moe_mm_data(
|
||||
topk_ids,
|
||||
expert_offsets,
|
||||
problem_sizes1,
|
||||
problem_sizes2,
|
||||
a_map,
|
||||
c_map,
|
||||
num_local_experts,
|
||||
n,
|
||||
k,
|
||||
)
|
||||
|
||||
c1 = torch.empty((m * topk, n * 2), device=device, dtype=torch.bfloat16)
|
||||
c2 = torch.empty((m * topk, k), device=device, dtype=torch.bfloat16)
|
||||
|
||||
@@ -323,3 +323,63 @@ def trtllm_fp8_per_tensor_scale_moe_wrapper(
|
||||
kwargs["activation_type"] = ActivationType(activation_type)
|
||||
|
||||
return trtllm_fp8_per_tensor_scale_moe(**kwargs)
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["output"])
|
||||
def trtllm_fp8_per_tensor_scale_routed_moe_out_wrapper(
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: Optional[torch.Tensor],
|
||||
hidden_states: torch.Tensor,
|
||||
gemm1_weights: torch.Tensor,
|
||||
output1_scales_scalar: torch.Tensor,
|
||||
output1_scales_gate_scalar: torch.Tensor,
|
||||
gemm2_weights: torch.Tensor,
|
||||
output2_scales_scalar: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
num_experts: int,
|
||||
top_k: int,
|
||||
intermediate_size: int,
|
||||
local_expert_offset: int,
|
||||
local_num_experts: int,
|
||||
routed_scaling_factor: Optional[float],
|
||||
routing_method_type: int,
|
||||
enable_pdl: Optional[bool] = None,
|
||||
tune_max_num_tokens: int = 8192,
|
||||
activation_type: Optional[int] = None,
|
||||
) -> None:
|
||||
try:
|
||||
from flashinfer.fused_moe import trtllm_fp8_per_tensor_scale_routed_moe
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"FlashInfer TRT-LLM per-tensor FP8 with materialized routing requires "
|
||||
"trtllm_fp8_per_tensor_scale_routed_moe; use the pinned CUDA FlashInfer build."
|
||||
) from e
|
||||
|
||||
kwargs = dict(
|
||||
topk_ids=topk_ids if topk_weights is None else (topk_ids, topk_weights),
|
||||
routing_bias=None,
|
||||
hidden_states=hidden_states,
|
||||
gemm1_weights=gemm1_weights,
|
||||
output1_scales_scalar=output1_scales_scalar,
|
||||
output1_scales_gate_scalar=output1_scales_gate_scalar,
|
||||
gemm2_weights=gemm2_weights,
|
||||
output2_scales_scalar=output2_scales_scalar,
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=intermediate_size,
|
||||
local_expert_offset=local_expert_offset,
|
||||
local_num_experts=local_num_experts,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
use_routing_scales_on_input=False,
|
||||
routing_method_type=routing_method_type,
|
||||
output=output,
|
||||
enable_pdl=enable_pdl,
|
||||
tune_max_num_tokens=tune_max_num_tokens,
|
||||
)
|
||||
if activation_type is not None:
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
|
||||
kwargs["activation_type"] = ActivationType(activation_type)
|
||||
trtllm_fp8_per_tensor_scale_routed_moe(**kwargs)
|
||||
|
||||
@@ -604,7 +604,10 @@ class FusedMoE(torch.nn.Module):
|
||||
_fuses_routed_scaling_factor_in_topk(self.quant_method, self.runner_backend)
|
||||
)
|
||||
|
||||
self.routing_method_type = routing_method_type
|
||||
self.routing_method_type = self.quant_method.get_moe_routing_method_type(
|
||||
self, routing_method_type
|
||||
)
|
||||
self.moe_runner_config.routing_method_type = self.routing_method_type
|
||||
|
||||
# overlap args
|
||||
self.down_gemm_overlap_args: Optional[DownGemmOverlapArgs] = None
|
||||
@@ -1564,11 +1567,11 @@ class FusedMoE(torch.nn.Module):
|
||||
|
||||
return forward_fuseep(self, hidden_states, topk_output)
|
||||
if is_in_tc_piecewise_cuda_graph():
|
||||
if (
|
||||
TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
and topk_output.expert_location_dispatch_info is not None
|
||||
if TopKOutputChecker.format_is_bypassed(topk_output) and (
|
||||
topk_output.expert_location_dispatch_info is not None
|
||||
or topk_output.num_token_non_padded is not None
|
||||
):
|
||||
# The opaque logits-only op cannot carry placement metadata.
|
||||
# The opaque logits-only op cannot carry padding or placement metadata.
|
||||
return self.forward_impl(
|
||||
hidden_states, topk_output, pre_quant_input=pre_quant_input
|
||||
)
|
||||
|
||||
@@ -275,6 +275,9 @@ class KTEPWrapperMethod(FusedMoEMethodBase):
|
||||
def get_moe_runner_backend(self, layer):
|
||||
return self.gpu_method.get_moe_runner_backend(layer)
|
||||
|
||||
def get_moe_routing_method_type(self, layer, requested_method):
|
||||
return self.gpu_method.get_moe_routing_method_type(layer, requested_method)
|
||||
|
||||
def submit(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -344,6 +347,12 @@ class KTEPWrapperMethod(FusedMoEMethodBase):
|
||||
Combined computation results from CPU and GPU experts
|
||||
"""
|
||||
from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput
|
||||
from sglang.srt.layers.moe.topk import TopKOutputChecker
|
||||
|
||||
if TopKOutputChecker.format_is_bypassed(dispatch_output.topk_output):
|
||||
dispatch_output = dispatch_output._replace(
|
||||
topk_output=dispatch_output.topk_output.to_standard()
|
||||
)
|
||||
|
||||
x = dispatch_output.hidden_states
|
||||
topk_output = dispatch_output.topk_output
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.layers.moe.flashinfer_trtllm_moe import (
|
||||
trtllm_fp8_block_scale_moe_out_wrapper,
|
||||
trtllm_fp8_block_scale_routed_moe_out_wrapper,
|
||||
trtllm_fp8_per_tensor_scale_moe_wrapper,
|
||||
trtllm_fp8_per_tensor_scale_routed_moe_out_wrapper,
|
||||
)
|
||||
from sglang.srt.layers.moe.moe_runner.base import (
|
||||
MoeQuantInfo,
|
||||
@@ -103,7 +104,6 @@ def _make_deferred_finalize_output(
|
||||
result,
|
||||
*,
|
||||
top_k: int,
|
||||
materialized_routing: bool = False,
|
||||
) -> FlashInferTrtllmDeferredFinalizeOutput:
|
||||
"""Validate and adapt FlashInfer's ``do_finalize=False`` output ABI."""
|
||||
gemm2_out, expert_weights, expanded_idx_to_permuted_idx = result[:3]
|
||||
@@ -178,18 +178,24 @@ def _is_gated(layer: Module) -> bool:
|
||||
FlashInferRouting = torch.Tensor | tuple[torch.Tensor, torch.Tensor]
|
||||
|
||||
|
||||
def _get_routing_for_flashinfer_routed(topk_output) -> FlashInferRouting:
|
||||
"""Return the `topk_ids` kernel argument for the trtllm routed MoEs."""
|
||||
def _get_routing_for_flashinfer_routed(
|
||||
topk_output, *, promote_half_scores: bool = False
|
||||
) -> FlashInferRouting:
|
||||
"""Return routed IDs/scores, optionally widening FP16 for native FP8 kernels."""
|
||||
from sglang.srt.layers.moe.topk import TopKOutputChecker
|
||||
|
||||
if TopKOutputChecker.format_is_packed(topk_output):
|
||||
return topk_output.packed_topk_ids
|
||||
|
||||
assert TopKOutputChecker.format_is_standard(topk_output)
|
||||
return (
|
||||
topk_output.topk_ids.contiguous(),
|
||||
topk_output.topk_weights.contiguous(),
|
||||
)
|
||||
weights = topk_output.topk_weights
|
||||
if promote_half_scores and weights.dtype == torch.float16:
|
||||
# FP8 providers accept BF16/FP32 scores. Widen and make contiguous in
|
||||
# one copy, retaining the values already rounded by the router.
|
||||
weights = weights.to(dtype=torch.float32, memory_format=torch.contiguous_format)
|
||||
else:
|
||||
weights = weights.contiguous()
|
||||
return topk_output.topk_ids.contiguous(), weights
|
||||
|
||||
|
||||
def _routing_top_k(routing: FlashInferRouting) -> int:
|
||||
@@ -771,6 +777,85 @@ class FlashInferTrtllmFp8MoeQuantInfo(MoeQuantInfo):
|
||||
activation_type: int | None = None
|
||||
|
||||
|
||||
def _run_fp8_per_tensor_scale_moe(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_output,
|
||||
quant_info: FlashInferTrtllmFp8MoeQuantInfo,
|
||||
runner_config: MoeRunnerConfig,
|
||||
use_routed_topk: bool,
|
||||
) -> torch.Tensor:
|
||||
from sglang.srt.layers.moe.topk import TopKOutputChecker
|
||||
from sglang.srt.layers.moe.utils import RoutingMethodType
|
||||
|
||||
assert quant_info.w13_input_scale is not None
|
||||
assert quant_info.output1_scales_scalar is not None
|
||||
assert quant_info.output1_scales_gate_scalar is not None
|
||||
assert quant_info.output2_scales_scalar is not None
|
||||
a_q, _ = scaled_fp8_quant(hidden_states, quant_info.w13_input_scale)
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group, disabled=not is_allocation_symmetric()
|
||||
):
|
||||
output = torch.empty(
|
||||
hidden_states.shape[0],
|
||||
hidden_states.shape[1],
|
||||
dtype=torch.bfloat16,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
common = dict(
|
||||
hidden_states=a_q,
|
||||
gemm1_weights=quant_info.w13_weight,
|
||||
output1_scales_scalar=quant_info.output1_scales_scalar,
|
||||
output1_scales_gate_scalar=quant_info.output1_scales_gate_scalar,
|
||||
gemm2_weights=quant_info.w2_weight,
|
||||
output2_scales_scalar=quant_info.output2_scales_scalar,
|
||||
num_experts=quant_info.global_num_experts,
|
||||
intermediate_size=int(quant_info.w2_weight.shape[2]),
|
||||
local_expert_offset=quant_info.local_expert_offset,
|
||||
local_num_experts=quant_info.local_num_experts,
|
||||
routed_scaling_factor=(
|
||||
runner_config.routed_scaling_factor
|
||||
if runner_config.routed_scaling_factor is not None
|
||||
else 1.0
|
||||
),
|
||||
tune_max_num_tokens=next_power_of_2(a_q.shape[0]),
|
||||
activation_type=quant_info.activation_type,
|
||||
)
|
||||
if use_routed_topk:
|
||||
routing = _get_routing_for_flashinfer_routed(
|
||||
topk_output, promote_half_scores=True
|
||||
)
|
||||
ids, weights = routing if isinstance(routing, tuple) else (routing, None)
|
||||
trtllm_fp8_per_tensor_scale_routed_moe_out_wrapper(
|
||||
topk_ids=ids,
|
||||
topk_weights=weights,
|
||||
output=output,
|
||||
top_k=_routing_top_k(routing),
|
||||
# Llama4's native FP8 body also consumes route scores as token scales.
|
||||
routing_method_type=(
|
||||
RoutingMethodType.Llama4
|
||||
if quant_info.routing_method_type == RoutingMethodType.Llama4
|
||||
else RoutingMethodType.TopK
|
||||
),
|
||||
**common,
|
||||
)
|
||||
else:
|
||||
assert TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
config = topk_output.topk_config
|
||||
result = trtllm_fp8_per_tensor_scale_moe_wrapper(
|
||||
routing_logits=topk_output.router_logits.to(torch.bfloat16),
|
||||
routing_bias=config.correction_bias,
|
||||
top_k=config.top_k,
|
||||
n_group=config.num_expert_group,
|
||||
topk_group=config.topk_group,
|
||||
use_routing_scales_on_input=False,
|
||||
routing_method_type=quant_info.routing_method_type,
|
||||
**common,
|
||||
)
|
||||
output.copy_(result)
|
||||
return output
|
||||
|
||||
|
||||
def fused_experts_none_to_flashinfer_trtllm_fp8(
|
||||
dispatch_output: StandardDispatchOutput | FlashinferDispatchOutput,
|
||||
quant_info: FlashInferTrtllmFp8MoeQuantInfo,
|
||||
@@ -800,6 +885,11 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
|
||||
else hidden_states.dtype
|
||||
)
|
||||
topk_output = dispatch_output.topk_output
|
||||
if TopKOutputChecker.format_is_bypassed(topk_output) and (
|
||||
topk_output.num_token_non_padded is not None
|
||||
or topk_output.expert_location_dispatch_info is not None
|
||||
):
|
||||
topk_output = topk_output.to_standard()
|
||||
materialized_topk = not use_routed_topk and TopKOutputChecker.format_is_standard(
|
||||
topk_output
|
||||
)
|
||||
@@ -823,15 +913,8 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
|
||||
)
|
||||
use_shuffled_weight = quant_info.use_mxfp8
|
||||
defer_finalize = _deferred_finalize_enabled.get()
|
||||
if defer_finalize and (
|
||||
not quant_info.block_quant
|
||||
or use_routed_topk
|
||||
or not TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
):
|
||||
raise RuntimeError(
|
||||
"FP8 deferred finalize requires block quantization, the logits-based "
|
||||
"FlashInfer TRTLLM backend, and bypassed TopK"
|
||||
)
|
||||
if defer_finalize and not quant_info.block_quant:
|
||||
raise RuntimeError("FP8 deferred finalize requires block quantization")
|
||||
|
||||
if quant_info.block_quant:
|
||||
assert quant_info.weight_block_k is not None
|
||||
@@ -884,17 +967,15 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
|
||||
# during torch.compile for piecewise cuda graph.
|
||||
# Use custom op wrapper for torch.compile compatibility.
|
||||
if use_routed_topk:
|
||||
assert runner_config.top_k is not None, (
|
||||
"runner_config.top_k is required for flashinfer_trtllm_routed."
|
||||
routing = _get_routing_for_flashinfer_routed(
|
||||
topk_output, promote_half_scores=True
|
||||
)
|
||||
routing = _get_routing_for_flashinfer_routed(topk_output)
|
||||
routed_top_k = _routing_top_k(routing)
|
||||
topk_ids, topk_weights = (
|
||||
routing if isinstance(routing, tuple) else (routing, None)
|
||||
)
|
||||
|
||||
trtllm_fp8_block_scale_routed_moe_out_wrapper(
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
routed_kwargs = dict(
|
||||
routing_bias=None,
|
||||
hidden_states=a_q,
|
||||
hidden_states_scale=a_sf_t,
|
||||
@@ -906,7 +987,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
|
||||
gemm2_weights=quant_info.w2_weight,
|
||||
gemm2_weights_scale=quant_info.w2_weight_scale_inv,
|
||||
num_experts=quant_info.global_num_experts,
|
||||
top_k=runner_config.top_k,
|
||||
top_k=routed_top_k,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=quant_info.intermediate_size,
|
||||
@@ -923,12 +1004,38 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
|
||||
else routing_method_type
|
||||
),
|
||||
use_shuffled_weight=use_shuffled_weight,
|
||||
output=symm_output,
|
||||
tune_max_num_tokens=next_power_of_2(a_q.shape[0]),
|
||||
fp8_quantization_type=int(fp8_quantization_type),
|
||||
activation_type=quant_info.activation_type,
|
||||
)
|
||||
output = cast(torch.Tensor, symm_output)
|
||||
if defer_finalize:
|
||||
from flashinfer.fused_moe import trtllm_fp8_block_scale_routed_moe
|
||||
|
||||
deferred_kwargs = dict(
|
||||
**routed_kwargs,
|
||||
topk_ids=routing,
|
||||
do_finalize=False,
|
||||
fp8_quantization_type=fp8_quantization_type,
|
||||
enable_pdl=a_q.shape[0] <= _TRTLLM_MOE_PDL_MAX_TOKENS,
|
||||
)
|
||||
if quant_info.activation_type is not None:
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
|
||||
deferred_kwargs["activation_type"] = ActivationType(
|
||||
quant_info.activation_type
|
||||
)
|
||||
output = _make_deferred_finalize_output(
|
||||
trtllm_fp8_block_scale_routed_moe(**deferred_kwargs),
|
||||
top_k=routed_top_k,
|
||||
)
|
||||
else:
|
||||
trtllm_fp8_block_scale_routed_moe_out_wrapper(
|
||||
**routed_kwargs,
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
output=symm_output,
|
||||
fp8_quantization_type=int(fp8_quantization_type),
|
||||
activation_type=quant_info.activation_type,
|
||||
)
|
||||
output = cast(torch.Tensor, symm_output)
|
||||
else:
|
||||
assert TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
common_kwargs = dict(
|
||||
@@ -987,59 +1094,9 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
|
||||
)
|
||||
output = cast(torch.Tensor, symm_output)
|
||||
else:
|
||||
assert TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
assert quant_info.w13_input_scale is not None
|
||||
assert quant_info.output1_scales_scalar is not None
|
||||
assert quant_info.output1_scales_gate_scalar is not None
|
||||
assert quant_info.output2_scales_scalar is not None
|
||||
|
||||
a_q, _ = scaled_fp8_quant(hidden_states, quant_info.w13_input_scale)
|
||||
|
||||
# Allocate output inside symmetric memory context
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group, disabled=not is_allocation_symmetric()
|
||||
):
|
||||
symm_output = torch.empty(
|
||||
hidden_states.shape[0],
|
||||
hidden_states.shape[1],
|
||||
dtype=torch.bfloat16,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
# Move kernel call outside context manager to avoid graph breaks
|
||||
# during torch.compile for piecewise cuda graph.
|
||||
# Use custom op wrapper for torch.compile compatibility.
|
||||
|
||||
router_logits = router_logits.to(torch.bfloat16)
|
||||
|
||||
output = trtllm_fp8_per_tensor_scale_moe_wrapper(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=correction_bias,
|
||||
hidden_states=a_q,
|
||||
gemm1_weights=quant_info.w13_weight,
|
||||
output1_scales_scalar=quant_info.output1_scales_scalar,
|
||||
output1_scales_gate_scalar=quant_info.output1_scales_gate_scalar,
|
||||
gemm2_weights=quant_info.w2_weight,
|
||||
output2_scales_scalar=quant_info.output2_scales_scalar,
|
||||
num_experts=quant_info.global_num_experts,
|
||||
top_k=topk_config.top_k,
|
||||
n_group=topk_config.num_expert_group,
|
||||
topk_group=topk_config.topk_group,
|
||||
intermediate_size=int(quant_info.w2_weight.shape[2]),
|
||||
local_expert_offset=quant_info.local_expert_offset,
|
||||
local_num_experts=quant_info.local_num_experts,
|
||||
routed_scaling_factor=(
|
||||
runner_config.routed_scaling_factor
|
||||
if runner_config.routed_scaling_factor is not None
|
||||
else 1.0
|
||||
),
|
||||
use_routing_scales_on_input=False,
|
||||
routing_method_type=routing_method_type,
|
||||
tune_max_num_tokens=next_power_of_2(a_q.shape[0]),
|
||||
activation_type=quant_info.activation_type,
|
||||
output = _run_fp8_per_tensor_scale_moe(
|
||||
hidden_states, topk_output, quant_info, runner_config, use_routed_topk
|
||||
)
|
||||
symm_output.copy_(output)
|
||||
output = symm_output
|
||||
|
||||
return StandardCombineInput(hidden_states=output)
|
||||
|
||||
@@ -1534,7 +1591,6 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
result = _make_deferred_finalize_output(
|
||||
result,
|
||||
top_k=_routing_top_k(routing),
|
||||
materialized_routing=isinstance(routing, tuple),
|
||||
)
|
||||
else:
|
||||
result = result[0]
|
||||
@@ -1724,7 +1780,6 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
|
||||
final_hidden_states = _make_deferred_finalize_output(
|
||||
final_hidden_states,
|
||||
top_k=runner_config.top_k,
|
||||
materialized_routing=isinstance(routing, tuple),
|
||||
)
|
||||
else:
|
||||
assert TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
|
||||
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeQuantInfo
|
||||
from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo
|
||||
from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput
|
||||
from sglang.srt.layers.moe.utils import MoeRunnerBackendLike
|
||||
from sglang.srt.layers.moe.utils import MoeRunnerBackendLike, RoutingMethodType
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
|
||||
|
||||
@@ -123,6 +123,12 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
"""
|
||||
return self.runner.runner_backend if self.runner is not None else None
|
||||
|
||||
def get_moe_routing_method_type(
|
||||
self, layer: torch.nn.Module, requested_method: RoutingMethodType | None
|
||||
) -> RoutingMethodType | None:
|
||||
"""Return the routing policy consumed by the selected execution path."""
|
||||
return requested_method
|
||||
|
||||
@abstractmethod
|
||||
def apply(
|
||||
self,
|
||||
|
||||
@@ -2978,6 +2978,18 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
return self.runner.runner_backend
|
||||
return self._moe_runner_backend
|
||||
|
||||
def get_moe_routing_method_type(self, layer, requested_method):
|
||||
backend = self.get_moe_runner_backend(layer)
|
||||
if (
|
||||
requested_method is None
|
||||
and backend is not None
|
||||
and (
|
||||
backend.is_flashinfer_trtllm() or backend.is_flashinfer_trtllm_routed()
|
||||
)
|
||||
):
|
||||
return RoutingMethodType.DeepSeekV3
|
||||
return requested_method
|
||||
|
||||
def get_triton_quant_info(self, layer: torch.nn.Module) -> TritonMoeQuantInfo:
|
||||
use_rocm_mxfp8 = self.use_mxfp8 and _is_hip and _is_gfx95_supported
|
||||
return TritonMoeQuantInfo(
|
||||
@@ -3270,10 +3282,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
is_gated=self.moe_runner_config.is_gated,
|
||||
)
|
||||
|
||||
routing_method_type = layer.routing_method_type
|
||||
if routing_method_type is None:
|
||||
routing_method_type = RoutingMethodType.DeepSeekV3
|
||||
|
||||
quant_info = FlashInferTrtllmFp8MoeQuantInfo(
|
||||
w13_weight=layer.w13_weight,
|
||||
w2_weight=layer.w2_weight,
|
||||
@@ -3281,7 +3289,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
local_expert_offset=moe_ep_rank * num_local_experts,
|
||||
local_num_experts=num_local_experts,
|
||||
intermediate_size=layer.w2_weight.shape[2],
|
||||
routing_method_type=int(routing_method_type),
|
||||
routing_method_type=int(layer.routing_method_type),
|
||||
block_quant=self.block_quant,
|
||||
use_mxfp8=getattr(self.quant_config, "use_mxfp8", False),
|
||||
weight_block_k=(
|
||||
|
||||
@@ -24,6 +24,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput
|
||||
from sglang.srt.layers.moe.utils import RoutingMethodType
|
||||
|
||||
# MXFP4 group/block size (E8M0 scale per 32 fp4 weights).
|
||||
_GROUP_SIZE = 32
|
||||
@@ -127,6 +128,11 @@ class Mxfp4FlashinferCutlassMoEMethod:
|
||||
def get_moe_runner_backend(self, layer):
|
||||
return self.runner.runner_backend
|
||||
|
||||
def get_moe_routing_method_type(
|
||||
self, layer: Module, requested_method: RoutingMethodType | None
|
||||
) -> RoutingMethodType | None:
|
||||
return requested_method
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
# Preserve the base FP4 post-load handling.
|
||||
self._fp8.process_weights_after_loading(layer)
|
||||
|
||||
@@ -168,6 +168,11 @@ class Mxfp4FlashinferTrtllmMoEMethod:
|
||||
def get_moe_runner_backend(self, layer):
|
||||
return MoeRunnerBackend.FLASHINFER_MXFP4
|
||||
|
||||
def get_moe_routing_method_type(
|
||||
self, layer: Module, requested_method: RoutingMethodType | None
|
||||
) -> RoutingMethodType | None:
|
||||
return requested_method
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer,
|
||||
|
||||
@@ -11,6 +11,7 @@ from sglang.srt.utils import log_info_on_rank0
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput
|
||||
from sglang.srt.layers.moe.utils import RoutingMethodType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -38,6 +39,11 @@ class Mxfp4HummingMoEMethod:
|
||||
def get_moe_runner_backend(self, layer):
|
||||
return self.runner.runner_backend
|
||||
|
||||
def get_moe_routing_method_type(
|
||||
self, layer: Module, requested_method: RoutingMethodType | None
|
||||
) -> RoutingMethodType | None:
|
||||
return requested_method
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: Module,
|
||||
|
||||
@@ -13,6 +13,7 @@ from sglang.srt.utils import log_info_on_rank0, round_up, set_weight_attrs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput
|
||||
from sglang.srt.layers.moe.utils import RoutingMethodType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -59,6 +60,11 @@ class Mxfp4MarlinMoEMethod:
|
||||
def get_moe_runner_backend(self, layer):
|
||||
return self.runner.runner_backend
|
||||
|
||||
def get_moe_routing_method_type(
|
||||
self, layer: Module, requested_method: RoutingMethodType | None
|
||||
) -> RoutingMethodType | None:
|
||||
return requested_method
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: Module,
|
||||
|
||||
@@ -312,7 +312,10 @@ class BailingMoE(nn.Module):
|
||||
self.topk.topk_config.apply_routed_scaling_factor_on_output = (
|
||||
self.experts.should_fuse_routed_scaling_factor_in_topk
|
||||
)
|
||||
self.topk.bind_runner_backend(self.experts.runner_backend)
|
||||
self.topk.bind_runner_backend(
|
||||
self.experts.runner_backend,
|
||||
routing_method_type=self.experts.routing_method_type,
|
||||
)
|
||||
|
||||
if self.num_shared_experts > 0:
|
||||
intermediate_size = self.intermediate_size * self.num_shared_experts
|
||||
|
||||
@@ -597,7 +597,10 @@ class BailingMoE(nn.Module):
|
||||
and getattr(quant_config, "is_fp4_experts", False)
|
||||
),
|
||||
)
|
||||
self.topk.bind_runner_backend(self.experts.runner_backend)
|
||||
self.topk.bind_runner_backend(
|
||||
self.experts.runner_backend,
|
||||
routing_method_type=self.experts.routing_method_type,
|
||||
)
|
||||
|
||||
# Whether to apply routed_scaling_factor at model layer.
|
||||
# For A2A MoE paths (e.g., DeepEP), the runner/post_permute does not apply it,
|
||||
|
||||
@@ -682,7 +682,10 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
|
||||
self.topk.topk_config.apply_routed_scaling_factor_on_output = (
|
||||
_is_npu or self.experts.should_fuse_routed_scaling_factor_in_topk
|
||||
)
|
||||
self.topk.bind_runner_backend(self.experts.runner_backend)
|
||||
self.topk.bind_runner_backend(
|
||||
self.experts.runner_backend,
|
||||
routing_method_type=self.experts.routing_method_type,
|
||||
)
|
||||
if self.use_block_routing:
|
||||
_require_block_routing_runner_compatibility(self.experts, self.top_k)
|
||||
# shared expert
|
||||
|
||||
Reference in New Issue
Block a user