[MoE] Skip block-padding rows in the TMA down activation and quant (#42212)

This commit is contained in:
WangHemeng
2026-10-12 11:10:30 +08:00
committed by GitHub
parent 1a7af3290d
commit 0c52cb1519
8 changed files with 568 additions and 49 deletions
@@ -54,6 +54,12 @@ struct ActivationParams {
const int32_t* __restrict__ expert_ids;
uint32_t expert_step;
float clamp_limit;
// Optional sorted-layout filtering (MoE TMA down path): a row is skipped if it
// is at or past *num_tokens_post_padded or its sorted_token_ids entry is the
// padding sentinel (>= num_valid_tokens). Must match token_mask in fused_moe_kernel.
const int32_t* __restrict__ sorted_token_ids;
const int32_t* __restrict__ num_tokens_post_padded;
uint32_t num_valid_tokens;
};
template <
@@ -63,7 +69,8 @@ template <
bool kFilterExpert,
bool kRoundActivation = false,
bool kReuseInput = false,
bool kClamp = false>
bool kClamp = false,
bool kFilterSorted = false>
__global__ void act_and_mul_kernel(const __grid_constant__ ActivationParams params) {
using namespace device;
constexpr auto kVecSize = kMaxVecBytes / sizeof(T);
@@ -73,7 +80,16 @@ __global__ void act_and_mul_kernel(const __grid_constant__ ActivationParams para
const auto token_id = tid / num_vecs;
if (token_id >= params.num_tokens) return;
if constexpr (kFilterExpert) {
// Like expert_ids, these are moe_align_block_size outputs the up GEMM already
// waited on (gdc_wait) before triggering this launch, so they may be read before
// PDLWaitPrimary. The host checks that both arrays cover every row, so the three
// loads are issued together instead of one after another.
if constexpr (kFilterSorted) {
const auto num_post = static_cast<uint32_t>(*params.num_tokens_post_padded);
const auto sorted_id = static_cast<uint32_t>(params.sorted_token_ids[token_id]);
const auto expert_id = params.expert_ids[token_id / params.expert_step];
if (token_id >= num_post || sorted_id >= params.num_valid_tokens || expert_id == -1) return;
} else if constexpr (kFilterExpert) {
if (params.expert_ids[token_id / params.expert_step] == -1) return;
}
const auto offset = tid % num_vecs;
@@ -140,21 +156,26 @@ struct ActivationKernel {
using kernel_fn_t = decltype(&act_and_mul_kernel<T, ActivationKind::kSiLU, kUsePDL, false>);
using unary_kernel_fn_t = decltype(&act_kernel<T, ActivationKind::kReLU2, kUsePDL>);
template <ActivationKind kAct, bool kFilterExpert, bool kRoundActivation = false, bool kReuseInput = false>
template <
ActivationKind kAct,
bool kFilterExpert,
bool kRoundActivation = false,
bool kReuseInput = false,
bool kFilterSorted = false>
static constexpr kernel_fn_t activation_kernel =
act_and_mul_kernel<T, kAct, kUsePDL, kFilterExpert, kRoundActivation, kReuseInput>;
act_and_mul_kernel<T, kAct, kUsePDL, kFilterExpert, kRoundActivation, kReuseInput, false, kFilterSorted>;
static_assert(device::kMaxVecBytes % sizeof(T) == 0, "unsupported data type");
template <bool kFilterExpert, bool kRoundActivation = false, bool kReuseInput = false>
template <bool kFilterExpert, bool kRoundActivation = false, bool kReuseInput = false, bool kFilterSorted = false>
static kernel_fn_t select_kernel(const std::string& type) {
using namespace host;
if (type == "silu") {
return activation_kernel<ActivationKind::kSiLU, kFilterExpert, kRoundActivation, kReuseInput>;
return activation_kernel<ActivationKind::kSiLU, kFilterExpert, kRoundActivation, kReuseInput, kFilterSorted>;
} else if (type == "gelu") {
return activation_kernel<ActivationKind::kGELU, kFilterExpert, kRoundActivation, kReuseInput>;
return activation_kernel<ActivationKind::kGELU, kFilterExpert, kRoundActivation, kReuseInput, kFilterSorted>;
} else if (type == "gelu_tanh") {
return activation_kernel<ActivationKind::kGELUTanh, kFilterExpert, kRoundActivation, kReuseInput>;
return activation_kernel<ActivationKind::kGELUTanh, kFilterExpert, kRoundActivation, kReuseInput, kFilterSorted>;
} else {
Panic("unsupported activation type: ", type);
}
@@ -168,7 +189,10 @@ struct ActivationKernel {
const std::string& type,
const int32_t* expert_ids,
uint32_t expert_step,
float clamp_limit = 0.0f) {
float clamp_limit = 0.0f,
const int32_t* sorted_token_ids = nullptr,
const int32_t* num_tokens_post_padded = nullptr,
uint32_t num_valid_tokens = 0) {
using namespace host;
auto N = SymbolicSize{"num_tokens"};
@@ -208,12 +232,24 @@ struct ActivationKernel {
.expert_ids = expert_ids,
.expert_step = expert_step,
.clamp_limit = clamp_limit,
.sorted_token_ids = sorted_token_ids,
.num_tokens_post_padded = num_tokens_post_padded,
.num_valid_tokens = num_valid_tokens,
};
if constexpr (kClamp) {
RuntimeCheck(type == "silu" && expert_ids == nullptr, "clamping requires unfiltered SiLU");
const auto kernel =
act_and_mul_kernel<T, ActivationKind::kSiLU, kUsePDL, false, kRoundActivation, kReuseInput, true>;
LaunchKernel(num_blocks, kBlockSize, device).enable_pdl(kUsePDL)(kernel, params);
} else if (sorted_token_ids != nullptr) {
// Only the plain (non-rounding, out-of-place) path takes the sorted
// layout; the sorted layout always comes with block-level expert_ids.
if constexpr (kRoundActivation || kReuseInput) {
Panic("sorted filtering is not supported for this activation variant");
} else {
const auto kernel = select_kernel<true, false, false, true>(type);
LaunchKernel(num_blocks, kBlockSize, device).enable_pdl(kUsePDL)(kernel, params);
}
} else if (expert_ids != nullptr) {
RuntimeCheck(expert_step > 0, "expert_step must be positive");
const auto kernel = select_kernel<true, kRoundActivation, kReuseInput>(type);
@@ -254,6 +290,41 @@ struct ActivationKernel {
launch(input, out, type, static_cast<const int32_t*>(expert_ids.data_ptr()), static_cast<uint32_t>(expert_step));
}
/// \brief Gated activation over the sorted, block-padded MoE layout.
/// \param expert_ids Per-block expert ids (-1 = non-local expert), one per expert_step rows.
/// \param sorted_token_ids moe_align_block_size output; entries >= num_valid_tokens are padding.
/// \param num_tokens_post_padded Device scalar; rows at or past it are padding.
/// Rows the MoE GEMM never reads are skipped and left unwritten in out.
static void run_activation_sorted_filtered(
const tvm::ffi::TensorView input,
const tvm::ffi::TensorView out,
const tvm::ffi::TensorView expert_ids,
int64_t expert_step,
const tvm::ffi::TensorView sorted_token_ids,
const tvm::ffi::TensorView num_tokens_post_padded,
int64_t num_valid_tokens,
std::string type) {
using namespace host;
CHECK_HOST(is_type<int32_t>(expert_ids.dtype())) << "expert_ids must have dtype int32";
CHECK_HOST(is_type<int32_t>(sorted_token_ids.dtype())) << "sorted_token_ids must have dtype int32";
CHECK_HOST(is_type<int32_t>(num_tokens_post_padded.dtype())) << "num_tokens_post_padded must have dtype int32";
CHECK_HOST(expert_step >= 1) << "expert_step must be positive, got " << expert_step;
CHECK_HOST(sorted_token_ids.numel() >= input.size(0)) << "sorted_token_ids must cover every row";
CHECK_HOST(expert_ids.numel() * expert_step >= input.size(0)) << "expert_ids must cover every row";
CHECK_HOST(0 <= num_valid_tokens && num_valid_tokens <= std::numeric_limits<int32_t>::max())
<< "num_valid_tokens out of range: " << num_valid_tokens;
launch(
input,
out,
type,
static_cast<const int32_t*>(expert_ids.data_ptr()),
static_cast<uint32_t>(expert_step),
/*clamp_limit=*/0.0f,
static_cast<const int32_t*>(sorted_token_ids.data_ptr()),
static_cast<const int32_t*>(num_tokens_post_padded.data_ptr()),
static_cast<uint32_t>(num_valid_tokens));
}
template <ActivationKind kAct>
static constexpr auto unary_kernel = act_kernel<T, kAct, kUsePDL>;
@@ -15,6 +15,8 @@
#include <algorithm>
#include <cstdint>
#include <cuda_fp8.h>
#include <limits>
#include <type_traits>
namespace sglang {
@@ -209,6 +211,29 @@ struct MaskedQuantKernelParams {
uint32_t masked_m_stride; // 1 (int32) or 2 (int64)
};
struct SortedQuantKernelParams {
QuantKernelParams base;
// Sorted, block-padded MoE layout (moe_align_block_size output): a row at or
// past *num_tokens_post_padded, or holding the padding sentinel
// (>= num_valid_tokens), is never read by the MoE GEMM. Must match token_mask
// in fused_moe_kernel.
const int32_t* __restrict__ sorted_token_ids;
const int32_t* __restrict__ num_tokens_post_padded;
uint32_t num_valid_tokens;
};
namespace detail {
SGL_DEVICE const QuantKernelParams& base_params(const QuantKernelParams& params) {
return params;
}
SGL_DEVICE const QuantKernelParams& base_params(const SortedQuantKernelParams& params) {
return params.base;
}
} // namespace detail
// PDL is a launch-scheduling knob, not a quant property, so it is a separate
// template parameter of the kernels/launchers rather than part of QuantTrait.
template <
@@ -392,12 +417,14 @@ struct QuantTrait {
};
// ---------------------------------------------------------------------------
// Flat schedule: one subwarp (kNumLanes) per group over a linear grid.
// Flat schedule: one subwarp (kNumLanes) per group over a linear grid. With
// kSortedRows, rows are in the sorted MoE layout and padding rows are skipped.
// ---------------------------------------------------------------------------
template <typename Trait, bool kUsePDL>
template <typename Trait, bool kUsePDL, bool kSortedRows = false>
__global__ __launch_bounds__(Trait::kBlockSize) void per_token_group_quant_flat_kernel(
const __grid_constant__ QuantKernelParams params) {
const __grid_constant__ std::conditional_t<kSortedRows, SortedQuantKernelParams, QuantKernelParams> kernel_params) {
using namespace device;
const QuantKernelParams& params = detail::base_params(kernel_params);
constexpr uint32_t kNumLanes = Trait::kNumLanes;
constexpr uint32_t kWorkPerWarp = kWarpThreads / kNumLanes;
const auto num_groups = params.scale.num_groups;
@@ -412,6 +439,16 @@ __global__ __launch_bounds__(Trait::kBlockSize) void per_token_group_quant_flat_
const auto token_idx = work_id / num_groups;
const auto group_idx = work_id % num_groups;
PDLWaitPrimary<kUsePDL>();
if constexpr (kSortedRows) {
// run() reduces across the whole warp, so only warps whose rows are all
// padding exit; a mixed warp also quantizes its padding rows (never read).
// The host checks sorted_token_ids covers every row, so both loads issue together.
const auto& sorted = kernel_params;
const auto num_post = static_cast<uint32_t>(*sorted.num_tokens_post_padded);
const auto sorted_id = static_cast<uint32_t>(sorted.sorted_token_ids[token_idx]);
const bool skip = token_idx >= num_post || sorted_id >= sorted.num_valid_tokens;
if (__all_sync(warp::kFullMask, skip)) return;
}
Trait::run(params, 0, token_idx, group_idx, lane_id);
PDLTriggerSecondary<kUsePDL>();
}
@@ -562,6 +599,17 @@ QuantHostContext<Trait> build_quant_context( //
};
}
template <typename Trait, bool kUsePDL, bool kSortedRows, typename Params>
void launch_flat_kernel(const QuantHostContext<Trait>& ctx, const Params& params) {
using namespace host;
const auto& p = ctx.params;
const int64_t total_threads = int64_t{p.num_tokens} * p.scale.num_groups * Trait::kNumLanes;
if (total_threads == 0) return;
const uint32_t num_blocks = div_ceil(total_threads, int64_t{Trait::kBlockSize});
LaunchKernel(num_blocks, Trait::kBlockSize, ctx.device)
.config({.use_pdl = kUsePDL})(per_token_group_quant_flat_kernel<Trait, kUsePDL, kSortedRows>, params);
}
template <
typename InputType,
typename QuantType,
@@ -575,14 +623,48 @@ struct PerTokenGroupQuantFlatKernel {
using Trait = QuantTrait<InputType, QuantType, kGroupSize, kUe8m0, kRowMajor, kAligned, kFuseSiluAndMul>;
static void run(tvm::ffi::TensorView input, tvm::ffi::TensorView output_q, tvm::ffi::TensorView output_s) {
using namespace host;
const auto ctx = build_quant_context<Trait, /*kMasked=*/false>(input, output_q, output_s);
const auto& p = ctx.params;
const int64_t total_threads = int64_t{p.num_tokens} * p.scale.num_groups * Trait::kNumLanes;
if (total_threads == 0) return;
const uint32_t num_blocks = div_ceil(total_threads, int64_t{Trait::kBlockSize});
LaunchKernel(num_blocks, Trait::kBlockSize, ctx.device)
.config({.use_pdl = kUsePDL})(per_token_group_quant_flat_kernel<Trait, kUsePDL>, p);
launch_flat_kernel<Trait, kUsePDL, /*kSortedRows=*/false>(ctx, ctx.params);
}
};
template <
typename InputType,
typename QuantType,
uint32_t kGroupSize,
bool kUe8m0,
bool kRowMajor,
bool kAligned,
bool kFuseSiluAndMul,
bool kUsePDL>
struct PerTokenGroupQuantSortedKernel {
using Trait = QuantTrait<InputType, QuantType, kGroupSize, kUe8m0, kRowMajor, kAligned, kFuseSiluAndMul>;
/// \brief Flat quantization over the sorted, block-padded MoE layout.
/// \param sorted_token_ids moe_align_block_size output; entries >= num_valid_tokens are padding.
/// \param num_tokens_post_padded Device scalar; rows at or past it are padding.
/// Rows the MoE GEMM never reads may be skipped and left unwritten.
static void
run(tvm::ffi::TensorView input,
tvm::ffi::TensorView output_q,
tvm::ffi::TensorView output_s,
tvm::ffi::TensorView sorted_token_ids,
tvm::ffi::TensorView num_tokens_post_padded,
int64_t num_valid_tokens) {
using namespace host;
CHECK_HOST(is_type<int32_t>(sorted_token_ids.dtype())) << "sorted_token_ids must have dtype int32";
CHECK_HOST(is_type<int32_t>(num_tokens_post_padded.dtype())) << "num_tokens_post_padded must have dtype int32";
CHECK_HOST(0 <= num_valid_tokens && num_valid_tokens <= std::numeric_limits<int32_t>::max())
<< "num_valid_tokens out of range: " << num_valid_tokens;
const auto ctx = build_quant_context<Trait, /*kMasked=*/false>(input, output_q, output_s);
CHECK_HOST(sorted_token_ids.numel() >= ctx.params.num_tokens) << "sorted_token_ids must cover every row";
const auto params = SortedQuantKernelParams{
ctx.params,
static_cast<const int32_t*>(sorted_token_ids.data_ptr()),
static_cast<const int32_t*>(num_tokens_post_padded.data_ptr()),
static_cast<uint32_t>(num_valid_tokens),
};
launch_flat_kernel<Trait, kUsePDL, /*kSortedRows=*/true>(ctx, params);
}
};
@@ -42,6 +42,10 @@ def activation_module(dtype: torch.dtype, *, fast_math: bool = True) -> Module:
"run_activation_filtered",
f"ActivationKernel<{args}>::run_activation_filtered",
),
(
"run_activation_sorted_filtered",
f"ActivationKernel<{args}>::run_activation_sorted_filtered",
),
(
"run_unary_activation",
f"ActivationKernel<{args}>::run_unary_activation",
@@ -102,12 +106,41 @@ def _run_activation_filtered_inplace(
module.run_activation_filtered(input_2d, out_2d, expert_ids, expert_step, op_name)
@register_custom_op(mutates_args=["out"])
def _run_activation_sorted_filtered_inplace(
op_name: str,
input: torch.Tensor,
out: torch.Tensor,
expert_ids: torch.Tensor,
expert_step: int,
sorted_token_ids: torch.Tensor,
num_tokens_post_padded: torch.Tensor,
num_valid_tokens: int,
) -> None:
hidden_size = input.shape[-1] // 2
module = activation_module(input.dtype)
module.run_activation_sorted_filtered(
input.view(-1, hidden_size * 2),
out.view(-1, hidden_size),
expert_ids,
expert_step,
sorted_token_ids,
num_tokens_post_padded,
num_valid_tokens,
op_name,
)
def run_activation(
op_name: str,
input: torch.Tensor,
out: Optional[torch.Tensor],
expert_ids: Optional[torch.Tensor] = None,
expert_step: int = 1,
*,
sorted_token_ids: Optional[torch.Tensor] = None,
num_tokens_post_padded: Optional[torch.Tensor] = None,
num_valid_tokens: int = 0,
) -> torch.Tensor:
"""Apply ``op_name`` activation followed by element-wise multiplication.
@@ -115,6 +148,12 @@ def run_activation(
routed expert id is ``-1``. ``expert_step`` is 1 for per-token routing and
``BLOCK_SIZE_M`` for sorted/TMA routing — i.e. ``expert_ids[token_id //
expert_step]`` is consulted before computing each row.
For the sorted, block-padded layout (``moe_align_block_size`` output with
``expert_step = BLOCK_SIZE_M``), also pass ``sorted_token_ids``,
``num_tokens_post_padded`` and ``num_valid_tokens`` (``topk_ids.numel()``)
to skip the padding rows the down GEMM never reads; skipped rows of ``out``
are left unwritten.
"""
assert op_name in SUPPORTED_ACTIVATIONS, f"Unsupported activation: {op_name}"
hidden_size = input.shape[-1] // 2
@@ -128,7 +167,21 @@ def run_activation(
# true dtype, so normalize here instead of asserting downstream.
if expert_ids.dtype != torch.int32:
expert_ids = expert_ids.to(torch.int32)
_run_activation_filtered_inplace(op_name, input, out, expert_ids, expert_step)
if sorted_token_ids is None:
_run_activation_filtered_inplace(
op_name, input, out, expert_ids, expert_step
)
else:
_run_activation_sorted_filtered_inplace(
op_name,
input,
out,
expert_ids,
expert_step,
sorted_token_ids,
num_tokens_post_padded,
num_valid_tokens,
)
return out
@@ -177,8 +230,21 @@ def silu_and_mul(
out: Optional[torch.Tensor] = None,
expert_ids: Optional[torch.Tensor] = None,
expert_step: int = 1,
*,
sorted_token_ids: Optional[torch.Tensor] = None,
num_tokens_post_padded: Optional[torch.Tensor] = None,
num_valid_tokens: int = 0,
) -> torch.Tensor:
return run_activation("silu", input, out, expert_ids, expert_step)
return run_activation(
"silu",
input,
out,
expert_ids,
expert_step,
sorted_token_ids=sorted_token_ids,
num_tokens_post_padded=num_tokens_post_padded,
num_valid_tokens=num_valid_tokens,
)
def silu_and_mul_with_activation_rounding(
@@ -216,8 +282,21 @@ def gelu_and_mul(
out: Optional[torch.Tensor] = None,
expert_ids: Optional[torch.Tensor] = None,
expert_step: int = 1,
*,
sorted_token_ids: Optional[torch.Tensor] = None,
num_tokens_post_padded: Optional[torch.Tensor] = None,
num_valid_tokens: int = 0,
) -> torch.Tensor:
return run_activation("gelu", input, out, expert_ids, expert_step)
return run_activation(
"gelu",
input,
out,
expert_ids,
expert_step,
sorted_token_ids=sorted_token_ids,
num_tokens_post_padded=num_tokens_post_padded,
num_valid_tokens=num_valid_tokens,
)
def gelu_tanh_and_mul(
@@ -225,5 +304,18 @@ def gelu_tanh_and_mul(
out: Optional[torch.Tensor] = None,
expert_ids: Optional[torch.Tensor] = None,
expert_step: int = 1,
*,
sorted_token_ids: Optional[torch.Tensor] = None,
num_tokens_post_padded: Optional[torch.Tensor] = None,
num_valid_tokens: int = 0,
) -> torch.Tensor:
return run_activation("gelu_tanh", input, out, expert_ids, expert_step)
return run_activation(
"gelu_tanh",
input,
out,
expert_ids,
expert_step,
sorted_token_ids=sorted_token_ids,
num_tokens_post_padded=num_tokens_post_padded,
num_valid_tokens=num_valid_tokens,
)
@@ -32,6 +32,7 @@ def _jit_module(
fuse_silu_and_mul: bool,
masked_layout: bool,
use_pdl: bool,
sorted_layout: bool = False,
) -> Module:
assert in_dtype in _SUPPORTED_INPUT_DTYPES
assert out_dtype in _SUPPORTED_OUTPUT_DTYPES
@@ -46,15 +47,13 @@ def _jit_module(
fuse_silu_and_mul,
use_pdl,
)
launcher = (
"PerTokenGroupQuantMaskedKernel"
if masked_layout
else "PerTokenGroupQuantFlatKernel"
)
assert not (masked_layout and sorted_layout)
layout = "masked" if masked_layout else ("sorted" if sorted_layout else "flat")
launcher = f"PerTokenGroupQuant{layout.capitalize()}Kernel"
return load_jit(
"per_token_group_quant",
*trait_args,
"masked" if masked_layout else "flat",
layout,
cuda_files=["gemm/per_token_group_quant.cuh"],
cuda_wrappers=[("per_token_group_quant", f"{launcher}<{trait_args}>::run")],
extra_cuda_cflags=["--use_fast_math"],
@@ -95,6 +94,9 @@ def _per_token_group_quant_custom_op(
fuse_silu_and_mul: bool = False,
masked_m: Optional[torch.Tensor] = None,
expected_m: Optional[int] = None,
sorted_token_ids: Optional[torch.Tensor] = None,
num_tokens_post_padded: Optional[torch.Tensor] = None,
num_valid_tokens: int = 0,
) -> None:
num_groups = output_q.shape[-1] // group_size
row_major, aligned = _infer_scale_layout(output_s, scale_ue8m0, num_groups)
@@ -108,11 +110,21 @@ def _per_token_group_quant_custom_op(
bool(fuse_silu_and_mul),
masked_m is not None,
is_arch_support_pdl(),
sorted_token_ids is not None,
)
if masked_m is not None:
module.per_token_group_quant(
input, output_q, output_s, masked_m, int(expected_m or -1)
)
elif sorted_token_ids is not None:
module.per_token_group_quant(
input,
output_q,
output_s,
sorted_token_ids,
num_tokens_post_padded,
num_valid_tokens,
)
else:
module.per_token_group_quant(input, output_q, output_s)
@@ -171,6 +183,9 @@ def per_token_group_quant(
*,
out_dtype: Optional[torch.dtype] = None,
column_major_scales: bool = False,
sorted_token_ids: Optional[torch.Tensor] = None,
num_tokens_post_padded: Optional[torch.Tensor] = None,
num_valid_tokens: int = 0,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Per-token-group quantization. Returns ``(output_q, output_s)``.
@@ -194,6 +209,11 @@ def per_token_group_quant(
``expected_m`` (masked only) is an optional expected-tokens-per-expert hint.
``sorted_token_ids`` selects the sorted layout: ``input`` rows follow
``moe_align_block_size`` (with ``num_tokens_post_padded`` and
``num_valid_tokens = topk_ids.numel()``), and rows the MoE GEMM never reads
may be skipped and left unwritten.
Inputs are bf16/fp16; group size is one of 16/32/64/128/256; the quant range
follows ``output_q.dtype`` (fp8_e4m3: +-448, int8: [-128, 127]).
"""
@@ -219,5 +239,8 @@ def per_token_group_quant(
fuse_silu_and_mul=fuse_silu_and_mul,
masked_m=masked_m,
expected_m=expected_m,
sorted_token_ids=sorted_token_ids,
num_tokens_post_padded=num_tokens_post_padded,
num_valid_tokens=num_valid_tokens,
)
return output_q, output_s
@@ -64,6 +64,9 @@ if _is_cuda:
silu_and_mul,
silu_and_mul_with_activation_rounding,
)
from sglang.kernels.ops.quantization.per_token_group_quant import (
per_token_group_quant,
)
elif _is_cpu and _is_cpu_amx_available:
pass
elif _is_hip:
@@ -125,6 +128,34 @@ def _validate_fused_swiglu_interleaved(
)
def _activation_row_filter(
*,
down_moe_use_tma: bool,
filter_expert: bool,
topk_ids: torch.Tensor,
expert_ids: torch.Tensor,
sorted_token_ids: torch.Tensor,
num_tokens_post_padded: torch.Tensor,
block_size_m: int,
) -> Dict[str, Any]:
"""Row filter for the CUDA JIT gated activations; empty computes every row."""
# HIP/XPU compute every row: the down kernel zeros filtered rows without
# reading their input.
if not _is_cuda or not (down_moe_use_tma or filter_expert):
return {}
if not down_moe_use_tma:
return dict(expert_ids=topk_ids.view(-1), expert_step=1)
# The TMA down buffer is sorted, block-padded and sized for the worst case;
# compute only the rows the down GEMM reads.
return dict(
expert_ids=expert_ids,
expert_step=block_size_m,
sorted_token_ids=sorted_token_ids,
num_tokens_post_padded=num_tokens_post_padded,
num_valid_tokens=topk_ids.numel(),
)
def _use_moe_sum_reduce_torch_compile(num_tokens: int) -> bool:
return num_tokens <= 32 and not is_batch_invariant_mode_enabled()
@@ -670,6 +701,16 @@ def _fused_moe_kernel_sequence(
dtype=hidden_states.dtype,
)
row_filter = _activation_row_filter(
down_moe_use_tma=down_moe_use_tma,
filter_expert=filter_expert,
topk_ids=topk_ids,
expert_ids=expert_ids,
sorted_token_ids=sorted_token_ids,
num_tokens_post_padded=num_tokens_post_padded,
block_size_m=config["BLOCK_SIZE_M"],
)
# Activation function with multiplication
if fuse_swiglu_interleaved:
# silu(gate) * up was already applied by the up-GEMM epilogue.
@@ -749,17 +790,9 @@ def _fused_moe_kernel_sequence(
swiglu_limit=swiglu_limit_for_triton,
)
elif _is_cuda or _is_hip or _is_xpu:
if filter_expert and _is_cuda:
# HIP/XPU fall through to the unfiltered path: the down kernel
# zeros filtered rows without reading their input.
silu_and_mul(
intermediate_cache1.view(-1, N),
intermediate_cache2,
expert_ids=(expert_ids if down_moe_use_tma else topk_ids.view(-1)),
expert_step=(config["BLOCK_SIZE_M"] if down_moe_use_tma else 1),
)
else:
silu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2)
silu_and_mul(
intermediate_cache1.view(-1, N), intermediate_cache2, **row_filter
)
elif _is_musa:
intermediate_cache2 = _silu_and_mul_musa(intermediate_cache1.view(-1, N))
else:
@@ -790,15 +823,9 @@ def _fused_moe_kernel_sequence(
activation_fn = (
gelu_tanh_and_mul if activation == "gelu_tanh" else gelu_and_mul
)
if filter_expert and _is_cuda:
activation_fn(
intermediate_cache1.view(-1, N),
intermediate_cache2,
expert_ids=(expert_ids if down_moe_use_tma else topk_ids.view(-1)),
expert_step=(config["BLOCK_SIZE_M"] if down_moe_use_tma else 1),
)
else:
activation_fn(intermediate_cache1.view(-1, N), intermediate_cache2)
activation_fn(
intermediate_cache1.view(-1, N), intermediate_cache2, **row_filter
)
else:
if _has_vllm_ops:
getattr(vllm_ops, f"{activation}_and_mul")(
@@ -845,6 +872,24 @@ def _fused_moe_kernel_sequence(
out_slice = out_hidden_states
out_slice.zero_()
if (
_is_cuda
and down_moe_use_tma
and use_fp8_w8a8
and block_shape is not None
and a2_scale is None
and intermediate_cache2.shape[-1] != block_shape[1]
):
# Same quant the down call would run, restricted to the rows the down
# GEMM reads; a single whole-row group keeps that call's per-token kernel.
intermediate_cache2, a2_scale = per_token_group_quant(
intermediate_cache2,
group_size=block_shape[1],
sorted_token_ids=sorted_token_ids,
num_tokens_post_padded=num_tokens_post_padded,
num_valid_tokens=topk_ids.numel(),
)
invoke_fused_moe_kernel(
intermediate_cache2,
w2,