[MoE] Skip block-padding rows in the TMA down activation and quant (#42212)
This commit is contained in:
@@ -54,6 +54,12 @@ struct ActivationParams {
|
||||
const int32_t* __restrict__ expert_ids;
|
||||
uint32_t expert_step;
|
||||
float clamp_limit;
|
||||
// Optional sorted-layout filtering (MoE TMA down path): a row is skipped if it
|
||||
// is at or past *num_tokens_post_padded or its sorted_token_ids entry is the
|
||||
// padding sentinel (>= num_valid_tokens). Must match token_mask in fused_moe_kernel.
|
||||
const int32_t* __restrict__ sorted_token_ids;
|
||||
const int32_t* __restrict__ num_tokens_post_padded;
|
||||
uint32_t num_valid_tokens;
|
||||
};
|
||||
|
||||
template <
|
||||
@@ -63,7 +69,8 @@ template <
|
||||
bool kFilterExpert,
|
||||
bool kRoundActivation = false,
|
||||
bool kReuseInput = false,
|
||||
bool kClamp = false>
|
||||
bool kClamp = false,
|
||||
bool kFilterSorted = false>
|
||||
__global__ void act_and_mul_kernel(const __grid_constant__ ActivationParams params) {
|
||||
using namespace device;
|
||||
constexpr auto kVecSize = kMaxVecBytes / sizeof(T);
|
||||
@@ -73,7 +80,16 @@ __global__ void act_and_mul_kernel(const __grid_constant__ ActivationParams para
|
||||
const auto token_id = tid / num_vecs;
|
||||
|
||||
if (token_id >= params.num_tokens) return;
|
||||
if constexpr (kFilterExpert) {
|
||||
// Like expert_ids, these are moe_align_block_size outputs the up GEMM already
|
||||
// waited on (gdc_wait) before triggering this launch, so they may be read before
|
||||
// PDLWaitPrimary. The host checks that both arrays cover every row, so the three
|
||||
// loads are issued together instead of one after another.
|
||||
if constexpr (kFilterSorted) {
|
||||
const auto num_post = static_cast<uint32_t>(*params.num_tokens_post_padded);
|
||||
const auto sorted_id = static_cast<uint32_t>(params.sorted_token_ids[token_id]);
|
||||
const auto expert_id = params.expert_ids[token_id / params.expert_step];
|
||||
if (token_id >= num_post || sorted_id >= params.num_valid_tokens || expert_id == -1) return;
|
||||
} else if constexpr (kFilterExpert) {
|
||||
if (params.expert_ids[token_id / params.expert_step] == -1) return;
|
||||
}
|
||||
const auto offset = tid % num_vecs;
|
||||
@@ -140,21 +156,26 @@ struct ActivationKernel {
|
||||
using kernel_fn_t = decltype(&act_and_mul_kernel<T, ActivationKind::kSiLU, kUsePDL, false>);
|
||||
using unary_kernel_fn_t = decltype(&act_kernel<T, ActivationKind::kReLU2, kUsePDL>);
|
||||
|
||||
template <ActivationKind kAct, bool kFilterExpert, bool kRoundActivation = false, bool kReuseInput = false>
|
||||
template <
|
||||
ActivationKind kAct,
|
||||
bool kFilterExpert,
|
||||
bool kRoundActivation = false,
|
||||
bool kReuseInput = false,
|
||||
bool kFilterSorted = false>
|
||||
static constexpr kernel_fn_t activation_kernel =
|
||||
act_and_mul_kernel<T, kAct, kUsePDL, kFilterExpert, kRoundActivation, kReuseInput>;
|
||||
act_and_mul_kernel<T, kAct, kUsePDL, kFilterExpert, kRoundActivation, kReuseInput, false, kFilterSorted>;
|
||||
|
||||
static_assert(device::kMaxVecBytes % sizeof(T) == 0, "unsupported data type");
|
||||
|
||||
template <bool kFilterExpert, bool kRoundActivation = false, bool kReuseInput = false>
|
||||
template <bool kFilterExpert, bool kRoundActivation = false, bool kReuseInput = false, bool kFilterSorted = false>
|
||||
static kernel_fn_t select_kernel(const std::string& type) {
|
||||
using namespace host;
|
||||
if (type == "silu") {
|
||||
return activation_kernel<ActivationKind::kSiLU, kFilterExpert, kRoundActivation, kReuseInput>;
|
||||
return activation_kernel<ActivationKind::kSiLU, kFilterExpert, kRoundActivation, kReuseInput, kFilterSorted>;
|
||||
} else if (type == "gelu") {
|
||||
return activation_kernel<ActivationKind::kGELU, kFilterExpert, kRoundActivation, kReuseInput>;
|
||||
return activation_kernel<ActivationKind::kGELU, kFilterExpert, kRoundActivation, kReuseInput, kFilterSorted>;
|
||||
} else if (type == "gelu_tanh") {
|
||||
return activation_kernel<ActivationKind::kGELUTanh, kFilterExpert, kRoundActivation, kReuseInput>;
|
||||
return activation_kernel<ActivationKind::kGELUTanh, kFilterExpert, kRoundActivation, kReuseInput, kFilterSorted>;
|
||||
} else {
|
||||
Panic("unsupported activation type: ", type);
|
||||
}
|
||||
@@ -168,7 +189,10 @@ struct ActivationKernel {
|
||||
const std::string& type,
|
||||
const int32_t* expert_ids,
|
||||
uint32_t expert_step,
|
||||
float clamp_limit = 0.0f) {
|
||||
float clamp_limit = 0.0f,
|
||||
const int32_t* sorted_token_ids = nullptr,
|
||||
const int32_t* num_tokens_post_padded = nullptr,
|
||||
uint32_t num_valid_tokens = 0) {
|
||||
using namespace host;
|
||||
|
||||
auto N = SymbolicSize{"num_tokens"};
|
||||
@@ -208,12 +232,24 @@ struct ActivationKernel {
|
||||
.expert_ids = expert_ids,
|
||||
.expert_step = expert_step,
|
||||
.clamp_limit = clamp_limit,
|
||||
.sorted_token_ids = sorted_token_ids,
|
||||
.num_tokens_post_padded = num_tokens_post_padded,
|
||||
.num_valid_tokens = num_valid_tokens,
|
||||
};
|
||||
if constexpr (kClamp) {
|
||||
RuntimeCheck(type == "silu" && expert_ids == nullptr, "clamping requires unfiltered SiLU");
|
||||
const auto kernel =
|
||||
act_and_mul_kernel<T, ActivationKind::kSiLU, kUsePDL, false, kRoundActivation, kReuseInput, true>;
|
||||
LaunchKernel(num_blocks, kBlockSize, device).enable_pdl(kUsePDL)(kernel, params);
|
||||
} else if (sorted_token_ids != nullptr) {
|
||||
// Only the plain (non-rounding, out-of-place) path takes the sorted
|
||||
// layout; the sorted layout always comes with block-level expert_ids.
|
||||
if constexpr (kRoundActivation || kReuseInput) {
|
||||
Panic("sorted filtering is not supported for this activation variant");
|
||||
} else {
|
||||
const auto kernel = select_kernel<true, false, false, true>(type);
|
||||
LaunchKernel(num_blocks, kBlockSize, device).enable_pdl(kUsePDL)(kernel, params);
|
||||
}
|
||||
} else if (expert_ids != nullptr) {
|
||||
RuntimeCheck(expert_step > 0, "expert_step must be positive");
|
||||
const auto kernel = select_kernel<true, kRoundActivation, kReuseInput>(type);
|
||||
@@ -254,6 +290,41 @@ struct ActivationKernel {
|
||||
launch(input, out, type, static_cast<const int32_t*>(expert_ids.data_ptr()), static_cast<uint32_t>(expert_step));
|
||||
}
|
||||
|
||||
/// \brief Gated activation over the sorted, block-padded MoE layout.
|
||||
/// \param expert_ids Per-block expert ids (-1 = non-local expert), one per expert_step rows.
|
||||
/// \param sorted_token_ids moe_align_block_size output; entries >= num_valid_tokens are padding.
|
||||
/// \param num_tokens_post_padded Device scalar; rows at or past it are padding.
|
||||
/// Rows the MoE GEMM never reads are skipped and left unwritten in out.
|
||||
static void run_activation_sorted_filtered(
|
||||
const tvm::ffi::TensorView input,
|
||||
const tvm::ffi::TensorView out,
|
||||
const tvm::ffi::TensorView expert_ids,
|
||||
int64_t expert_step,
|
||||
const tvm::ffi::TensorView sorted_token_ids,
|
||||
const tvm::ffi::TensorView num_tokens_post_padded,
|
||||
int64_t num_valid_tokens,
|
||||
std::string type) {
|
||||
using namespace host;
|
||||
CHECK_HOST(is_type<int32_t>(expert_ids.dtype())) << "expert_ids must have dtype int32";
|
||||
CHECK_HOST(is_type<int32_t>(sorted_token_ids.dtype())) << "sorted_token_ids must have dtype int32";
|
||||
CHECK_HOST(is_type<int32_t>(num_tokens_post_padded.dtype())) << "num_tokens_post_padded must have dtype int32";
|
||||
CHECK_HOST(expert_step >= 1) << "expert_step must be positive, got " << expert_step;
|
||||
CHECK_HOST(sorted_token_ids.numel() >= input.size(0)) << "sorted_token_ids must cover every row";
|
||||
CHECK_HOST(expert_ids.numel() * expert_step >= input.size(0)) << "expert_ids must cover every row";
|
||||
CHECK_HOST(0 <= num_valid_tokens && num_valid_tokens <= std::numeric_limits<int32_t>::max())
|
||||
<< "num_valid_tokens out of range: " << num_valid_tokens;
|
||||
launch(
|
||||
input,
|
||||
out,
|
||||
type,
|
||||
static_cast<const int32_t*>(expert_ids.data_ptr()),
|
||||
static_cast<uint32_t>(expert_step),
|
||||
/*clamp_limit=*/0.0f,
|
||||
static_cast<const int32_t*>(sorted_token_ids.data_ptr()),
|
||||
static_cast<const int32_t*>(num_tokens_post_padded.data_ptr()),
|
||||
static_cast<uint32_t>(num_valid_tokens));
|
||||
}
|
||||
|
||||
template <ActivationKind kAct>
|
||||
static constexpr auto unary_kernel = act_kernel<T, kAct, kUsePDL>;
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cuda_fp8.h>
|
||||
#include <limits>
|
||||
#include <type_traits>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
@@ -209,6 +211,29 @@ struct MaskedQuantKernelParams {
|
||||
uint32_t masked_m_stride; // 1 (int32) or 2 (int64)
|
||||
};
|
||||
|
||||
struct SortedQuantKernelParams {
|
||||
QuantKernelParams base;
|
||||
// Sorted, block-padded MoE layout (moe_align_block_size output): a row at or
|
||||
// past *num_tokens_post_padded, or holding the padding sentinel
|
||||
// (>= num_valid_tokens), is never read by the MoE GEMM. Must match token_mask
|
||||
// in fused_moe_kernel.
|
||||
const int32_t* __restrict__ sorted_token_ids;
|
||||
const int32_t* __restrict__ num_tokens_post_padded;
|
||||
uint32_t num_valid_tokens;
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
|
||||
SGL_DEVICE const QuantKernelParams& base_params(const QuantKernelParams& params) {
|
||||
return params;
|
||||
}
|
||||
|
||||
SGL_DEVICE const QuantKernelParams& base_params(const SortedQuantKernelParams& params) {
|
||||
return params.base;
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
// PDL is a launch-scheduling knob, not a quant property, so it is a separate
|
||||
// template parameter of the kernels/launchers rather than part of QuantTrait.
|
||||
template <
|
||||
@@ -392,12 +417,14 @@ struct QuantTrait {
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Flat schedule: one subwarp (kNumLanes) per group over a linear grid.
|
||||
// Flat schedule: one subwarp (kNumLanes) per group over a linear grid. With
|
||||
// kSortedRows, rows are in the sorted MoE layout and padding rows are skipped.
|
||||
// ---------------------------------------------------------------------------
|
||||
template <typename Trait, bool kUsePDL>
|
||||
template <typename Trait, bool kUsePDL, bool kSortedRows = false>
|
||||
__global__ __launch_bounds__(Trait::kBlockSize) void per_token_group_quant_flat_kernel(
|
||||
const __grid_constant__ QuantKernelParams params) {
|
||||
const __grid_constant__ std::conditional_t<kSortedRows, SortedQuantKernelParams, QuantKernelParams> kernel_params) {
|
||||
using namespace device;
|
||||
const QuantKernelParams& params = detail::base_params(kernel_params);
|
||||
constexpr uint32_t kNumLanes = Trait::kNumLanes;
|
||||
constexpr uint32_t kWorkPerWarp = kWarpThreads / kNumLanes;
|
||||
const auto num_groups = params.scale.num_groups;
|
||||
@@ -412,6 +439,16 @@ __global__ __launch_bounds__(Trait::kBlockSize) void per_token_group_quant_flat_
|
||||
const auto token_idx = work_id / num_groups;
|
||||
const auto group_idx = work_id % num_groups;
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
if constexpr (kSortedRows) {
|
||||
// run() reduces across the whole warp, so only warps whose rows are all
|
||||
// padding exit; a mixed warp also quantizes its padding rows (never read).
|
||||
// The host checks sorted_token_ids covers every row, so both loads issue together.
|
||||
const auto& sorted = kernel_params;
|
||||
const auto num_post = static_cast<uint32_t>(*sorted.num_tokens_post_padded);
|
||||
const auto sorted_id = static_cast<uint32_t>(sorted.sorted_token_ids[token_idx]);
|
||||
const bool skip = token_idx >= num_post || sorted_id >= sorted.num_valid_tokens;
|
||||
if (__all_sync(warp::kFullMask, skip)) return;
|
||||
}
|
||||
Trait::run(params, 0, token_idx, group_idx, lane_id);
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
@@ -562,6 +599,17 @@ QuantHostContext<Trait> build_quant_context( //
|
||||
};
|
||||
}
|
||||
|
||||
template <typename Trait, bool kUsePDL, bool kSortedRows, typename Params>
|
||||
void launch_flat_kernel(const QuantHostContext<Trait>& ctx, const Params& params) {
|
||||
using namespace host;
|
||||
const auto& p = ctx.params;
|
||||
const int64_t total_threads = int64_t{p.num_tokens} * p.scale.num_groups * Trait::kNumLanes;
|
||||
if (total_threads == 0) return;
|
||||
const uint32_t num_blocks = div_ceil(total_threads, int64_t{Trait::kBlockSize});
|
||||
LaunchKernel(num_blocks, Trait::kBlockSize, ctx.device)
|
||||
.config({.use_pdl = kUsePDL})(per_token_group_quant_flat_kernel<Trait, kUsePDL, kSortedRows>, params);
|
||||
}
|
||||
|
||||
template <
|
||||
typename InputType,
|
||||
typename QuantType,
|
||||
@@ -575,14 +623,48 @@ struct PerTokenGroupQuantFlatKernel {
|
||||
using Trait = QuantTrait<InputType, QuantType, kGroupSize, kUe8m0, kRowMajor, kAligned, kFuseSiluAndMul>;
|
||||
|
||||
static void run(tvm::ffi::TensorView input, tvm::ffi::TensorView output_q, tvm::ffi::TensorView output_s) {
|
||||
using namespace host;
|
||||
const auto ctx = build_quant_context<Trait, /*kMasked=*/false>(input, output_q, output_s);
|
||||
const auto& p = ctx.params;
|
||||
const int64_t total_threads = int64_t{p.num_tokens} * p.scale.num_groups * Trait::kNumLanes;
|
||||
if (total_threads == 0) return;
|
||||
const uint32_t num_blocks = div_ceil(total_threads, int64_t{Trait::kBlockSize});
|
||||
LaunchKernel(num_blocks, Trait::kBlockSize, ctx.device)
|
||||
.config({.use_pdl = kUsePDL})(per_token_group_quant_flat_kernel<Trait, kUsePDL>, p);
|
||||
launch_flat_kernel<Trait, kUsePDL, /*kSortedRows=*/false>(ctx, ctx.params);
|
||||
}
|
||||
};
|
||||
|
||||
template <
|
||||
typename InputType,
|
||||
typename QuantType,
|
||||
uint32_t kGroupSize,
|
||||
bool kUe8m0,
|
||||
bool kRowMajor,
|
||||
bool kAligned,
|
||||
bool kFuseSiluAndMul,
|
||||
bool kUsePDL>
|
||||
struct PerTokenGroupQuantSortedKernel {
|
||||
using Trait = QuantTrait<InputType, QuantType, kGroupSize, kUe8m0, kRowMajor, kAligned, kFuseSiluAndMul>;
|
||||
|
||||
/// \brief Flat quantization over the sorted, block-padded MoE layout.
|
||||
/// \param sorted_token_ids moe_align_block_size output; entries >= num_valid_tokens are padding.
|
||||
/// \param num_tokens_post_padded Device scalar; rows at or past it are padding.
|
||||
/// Rows the MoE GEMM never reads may be skipped and left unwritten.
|
||||
static void
|
||||
run(tvm::ffi::TensorView input,
|
||||
tvm::ffi::TensorView output_q,
|
||||
tvm::ffi::TensorView output_s,
|
||||
tvm::ffi::TensorView sorted_token_ids,
|
||||
tvm::ffi::TensorView num_tokens_post_padded,
|
||||
int64_t num_valid_tokens) {
|
||||
using namespace host;
|
||||
CHECK_HOST(is_type<int32_t>(sorted_token_ids.dtype())) << "sorted_token_ids must have dtype int32";
|
||||
CHECK_HOST(is_type<int32_t>(num_tokens_post_padded.dtype())) << "num_tokens_post_padded must have dtype int32";
|
||||
CHECK_HOST(0 <= num_valid_tokens && num_valid_tokens <= std::numeric_limits<int32_t>::max())
|
||||
<< "num_valid_tokens out of range: " << num_valid_tokens;
|
||||
const auto ctx = build_quant_context<Trait, /*kMasked=*/false>(input, output_q, output_s);
|
||||
CHECK_HOST(sorted_token_ids.numel() >= ctx.params.num_tokens) << "sorted_token_ids must cover every row";
|
||||
const auto params = SortedQuantKernelParams{
|
||||
ctx.params,
|
||||
static_cast<const int32_t*>(sorted_token_ids.data_ptr()),
|
||||
static_cast<const int32_t*>(num_tokens_post_padded.data_ptr()),
|
||||
static_cast<uint32_t>(num_valid_tokens),
|
||||
};
|
||||
launch_flat_kernel<Trait, kUsePDL, /*kSortedRows=*/true>(ctx, params);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -42,6 +42,10 @@ def activation_module(dtype: torch.dtype, *, fast_math: bool = True) -> Module:
|
||||
"run_activation_filtered",
|
||||
f"ActivationKernel<{args}>::run_activation_filtered",
|
||||
),
|
||||
(
|
||||
"run_activation_sorted_filtered",
|
||||
f"ActivationKernel<{args}>::run_activation_sorted_filtered",
|
||||
),
|
||||
(
|
||||
"run_unary_activation",
|
||||
f"ActivationKernel<{args}>::run_unary_activation",
|
||||
@@ -102,12 +106,41 @@ def _run_activation_filtered_inplace(
|
||||
module.run_activation_filtered(input_2d, out_2d, expert_ids, expert_step, op_name)
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["out"])
|
||||
def _run_activation_sorted_filtered_inplace(
|
||||
op_name: str,
|
||||
input: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
expert_step: int,
|
||||
sorted_token_ids: torch.Tensor,
|
||||
num_tokens_post_padded: torch.Tensor,
|
||||
num_valid_tokens: int,
|
||||
) -> None:
|
||||
hidden_size = input.shape[-1] // 2
|
||||
module = activation_module(input.dtype)
|
||||
module.run_activation_sorted_filtered(
|
||||
input.view(-1, hidden_size * 2),
|
||||
out.view(-1, hidden_size),
|
||||
expert_ids,
|
||||
expert_step,
|
||||
sorted_token_ids,
|
||||
num_tokens_post_padded,
|
||||
num_valid_tokens,
|
||||
op_name,
|
||||
)
|
||||
|
||||
|
||||
def run_activation(
|
||||
op_name: str,
|
||||
input: torch.Tensor,
|
||||
out: Optional[torch.Tensor],
|
||||
expert_ids: Optional[torch.Tensor] = None,
|
||||
expert_step: int = 1,
|
||||
*,
|
||||
sorted_token_ids: Optional[torch.Tensor] = None,
|
||||
num_tokens_post_padded: Optional[torch.Tensor] = None,
|
||||
num_valid_tokens: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Apply ``op_name`` activation followed by element-wise multiplication.
|
||||
|
||||
@@ -115,6 +148,12 @@ def run_activation(
|
||||
routed expert id is ``-1``. ``expert_step`` is 1 for per-token routing and
|
||||
``BLOCK_SIZE_M`` for sorted/TMA routing — i.e. ``expert_ids[token_id //
|
||||
expert_step]`` is consulted before computing each row.
|
||||
|
||||
For the sorted, block-padded layout (``moe_align_block_size`` output with
|
||||
``expert_step = BLOCK_SIZE_M``), also pass ``sorted_token_ids``,
|
||||
``num_tokens_post_padded`` and ``num_valid_tokens`` (``topk_ids.numel()``)
|
||||
to skip the padding rows the down GEMM never reads; skipped rows of ``out``
|
||||
are left unwritten.
|
||||
"""
|
||||
assert op_name in SUPPORTED_ACTIVATIONS, f"Unsupported activation: {op_name}"
|
||||
hidden_size = input.shape[-1] // 2
|
||||
@@ -128,7 +167,21 @@ def run_activation(
|
||||
# true dtype, so normalize here instead of asserting downstream.
|
||||
if expert_ids.dtype != torch.int32:
|
||||
expert_ids = expert_ids.to(torch.int32)
|
||||
_run_activation_filtered_inplace(op_name, input, out, expert_ids, expert_step)
|
||||
if sorted_token_ids is None:
|
||||
_run_activation_filtered_inplace(
|
||||
op_name, input, out, expert_ids, expert_step
|
||||
)
|
||||
else:
|
||||
_run_activation_sorted_filtered_inplace(
|
||||
op_name,
|
||||
input,
|
||||
out,
|
||||
expert_ids,
|
||||
expert_step,
|
||||
sorted_token_ids,
|
||||
num_tokens_post_padded,
|
||||
num_valid_tokens,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@@ -177,8 +230,21 @@ def silu_and_mul(
|
||||
out: Optional[torch.Tensor] = None,
|
||||
expert_ids: Optional[torch.Tensor] = None,
|
||||
expert_step: int = 1,
|
||||
*,
|
||||
sorted_token_ids: Optional[torch.Tensor] = None,
|
||||
num_tokens_post_padded: Optional[torch.Tensor] = None,
|
||||
num_valid_tokens: int = 0,
|
||||
) -> torch.Tensor:
|
||||
return run_activation("silu", input, out, expert_ids, expert_step)
|
||||
return run_activation(
|
||||
"silu",
|
||||
input,
|
||||
out,
|
||||
expert_ids,
|
||||
expert_step,
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
num_tokens_post_padded=num_tokens_post_padded,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
)
|
||||
|
||||
|
||||
def silu_and_mul_with_activation_rounding(
|
||||
@@ -216,8 +282,21 @@ def gelu_and_mul(
|
||||
out: Optional[torch.Tensor] = None,
|
||||
expert_ids: Optional[torch.Tensor] = None,
|
||||
expert_step: int = 1,
|
||||
*,
|
||||
sorted_token_ids: Optional[torch.Tensor] = None,
|
||||
num_tokens_post_padded: Optional[torch.Tensor] = None,
|
||||
num_valid_tokens: int = 0,
|
||||
) -> torch.Tensor:
|
||||
return run_activation("gelu", input, out, expert_ids, expert_step)
|
||||
return run_activation(
|
||||
"gelu",
|
||||
input,
|
||||
out,
|
||||
expert_ids,
|
||||
expert_step,
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
num_tokens_post_padded=num_tokens_post_padded,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
)
|
||||
|
||||
|
||||
def gelu_tanh_and_mul(
|
||||
@@ -225,5 +304,18 @@ def gelu_tanh_and_mul(
|
||||
out: Optional[torch.Tensor] = None,
|
||||
expert_ids: Optional[torch.Tensor] = None,
|
||||
expert_step: int = 1,
|
||||
*,
|
||||
sorted_token_ids: Optional[torch.Tensor] = None,
|
||||
num_tokens_post_padded: Optional[torch.Tensor] = None,
|
||||
num_valid_tokens: int = 0,
|
||||
) -> torch.Tensor:
|
||||
return run_activation("gelu_tanh", input, out, expert_ids, expert_step)
|
||||
return run_activation(
|
||||
"gelu_tanh",
|
||||
input,
|
||||
out,
|
||||
expert_ids,
|
||||
expert_step,
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
num_tokens_post_padded=num_tokens_post_padded,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
)
|
||||
|
||||
@@ -32,6 +32,7 @@ def _jit_module(
|
||||
fuse_silu_and_mul: bool,
|
||||
masked_layout: bool,
|
||||
use_pdl: bool,
|
||||
sorted_layout: bool = False,
|
||||
) -> Module:
|
||||
assert in_dtype in _SUPPORTED_INPUT_DTYPES
|
||||
assert out_dtype in _SUPPORTED_OUTPUT_DTYPES
|
||||
@@ -46,15 +47,13 @@ def _jit_module(
|
||||
fuse_silu_and_mul,
|
||||
use_pdl,
|
||||
)
|
||||
launcher = (
|
||||
"PerTokenGroupQuantMaskedKernel"
|
||||
if masked_layout
|
||||
else "PerTokenGroupQuantFlatKernel"
|
||||
)
|
||||
assert not (masked_layout and sorted_layout)
|
||||
layout = "masked" if masked_layout else ("sorted" if sorted_layout else "flat")
|
||||
launcher = f"PerTokenGroupQuant{layout.capitalize()}Kernel"
|
||||
return load_jit(
|
||||
"per_token_group_quant",
|
||||
*trait_args,
|
||||
"masked" if masked_layout else "flat",
|
||||
layout,
|
||||
cuda_files=["gemm/per_token_group_quant.cuh"],
|
||||
cuda_wrappers=[("per_token_group_quant", f"{launcher}<{trait_args}>::run")],
|
||||
extra_cuda_cflags=["--use_fast_math"],
|
||||
@@ -95,6 +94,9 @@ def _per_token_group_quant_custom_op(
|
||||
fuse_silu_and_mul: bool = False,
|
||||
masked_m: Optional[torch.Tensor] = None,
|
||||
expected_m: Optional[int] = None,
|
||||
sorted_token_ids: Optional[torch.Tensor] = None,
|
||||
num_tokens_post_padded: Optional[torch.Tensor] = None,
|
||||
num_valid_tokens: int = 0,
|
||||
) -> None:
|
||||
num_groups = output_q.shape[-1] // group_size
|
||||
row_major, aligned = _infer_scale_layout(output_s, scale_ue8m0, num_groups)
|
||||
@@ -108,11 +110,21 @@ def _per_token_group_quant_custom_op(
|
||||
bool(fuse_silu_and_mul),
|
||||
masked_m is not None,
|
||||
is_arch_support_pdl(),
|
||||
sorted_token_ids is not None,
|
||||
)
|
||||
if masked_m is not None:
|
||||
module.per_token_group_quant(
|
||||
input, output_q, output_s, masked_m, int(expected_m or -1)
|
||||
)
|
||||
elif sorted_token_ids is not None:
|
||||
module.per_token_group_quant(
|
||||
input,
|
||||
output_q,
|
||||
output_s,
|
||||
sorted_token_ids,
|
||||
num_tokens_post_padded,
|
||||
num_valid_tokens,
|
||||
)
|
||||
else:
|
||||
module.per_token_group_quant(input, output_q, output_s)
|
||||
|
||||
@@ -171,6 +183,9 @@ def per_token_group_quant(
|
||||
*,
|
||||
out_dtype: Optional[torch.dtype] = None,
|
||||
column_major_scales: bool = False,
|
||||
sorted_token_ids: Optional[torch.Tensor] = None,
|
||||
num_tokens_post_padded: Optional[torch.Tensor] = None,
|
||||
num_valid_tokens: int = 0,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-token-group quantization. Returns ``(output_q, output_s)``.
|
||||
|
||||
@@ -194,6 +209,11 @@ def per_token_group_quant(
|
||||
|
||||
``expected_m`` (masked only) is an optional expected-tokens-per-expert hint.
|
||||
|
||||
``sorted_token_ids`` selects the sorted layout: ``input`` rows follow
|
||||
``moe_align_block_size`` (with ``num_tokens_post_padded`` and
|
||||
``num_valid_tokens = topk_ids.numel()``), and rows the MoE GEMM never reads
|
||||
may be skipped and left unwritten.
|
||||
|
||||
Inputs are bf16/fp16; group size is one of 16/32/64/128/256; the quant range
|
||||
follows ``output_q.dtype`` (fp8_e4m3: +-448, int8: [-128, 127]).
|
||||
"""
|
||||
@@ -219,5 +239,8 @@ def per_token_group_quant(
|
||||
fuse_silu_and_mul=fuse_silu_and_mul,
|
||||
masked_m=masked_m,
|
||||
expected_m=expected_m,
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
num_tokens_post_padded=num_tokens_post_padded,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
)
|
||||
return output_q, output_s
|
||||
|
||||
@@ -64,6 +64,9 @@ if _is_cuda:
|
||||
silu_and_mul,
|
||||
silu_and_mul_with_activation_rounding,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.per_token_group_quant import (
|
||||
per_token_group_quant,
|
||||
)
|
||||
elif _is_cpu and _is_cpu_amx_available:
|
||||
pass
|
||||
elif _is_hip:
|
||||
@@ -125,6 +128,34 @@ def _validate_fused_swiglu_interleaved(
|
||||
)
|
||||
|
||||
|
||||
def _activation_row_filter(
|
||||
*,
|
||||
down_moe_use_tma: bool,
|
||||
filter_expert: bool,
|
||||
topk_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
sorted_token_ids: torch.Tensor,
|
||||
num_tokens_post_padded: torch.Tensor,
|
||||
block_size_m: int,
|
||||
) -> Dict[str, Any]:
|
||||
"""Row filter for the CUDA JIT gated activations; empty computes every row."""
|
||||
# HIP/XPU compute every row: the down kernel zeros filtered rows without
|
||||
# reading their input.
|
||||
if not _is_cuda or not (down_moe_use_tma or filter_expert):
|
||||
return {}
|
||||
if not down_moe_use_tma:
|
||||
return dict(expert_ids=topk_ids.view(-1), expert_step=1)
|
||||
# The TMA down buffer is sorted, block-padded and sized for the worst case;
|
||||
# compute only the rows the down GEMM reads.
|
||||
return dict(
|
||||
expert_ids=expert_ids,
|
||||
expert_step=block_size_m,
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
num_tokens_post_padded=num_tokens_post_padded,
|
||||
num_valid_tokens=topk_ids.numel(),
|
||||
)
|
||||
|
||||
|
||||
def _use_moe_sum_reduce_torch_compile(num_tokens: int) -> bool:
|
||||
return num_tokens <= 32 and not is_batch_invariant_mode_enabled()
|
||||
|
||||
@@ -670,6 +701,16 @@ def _fused_moe_kernel_sequence(
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
|
||||
row_filter = _activation_row_filter(
|
||||
down_moe_use_tma=down_moe_use_tma,
|
||||
filter_expert=filter_expert,
|
||||
topk_ids=topk_ids,
|
||||
expert_ids=expert_ids,
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
num_tokens_post_padded=num_tokens_post_padded,
|
||||
block_size_m=config["BLOCK_SIZE_M"],
|
||||
)
|
||||
|
||||
# Activation function with multiplication
|
||||
if fuse_swiglu_interleaved:
|
||||
# silu(gate) * up was already applied by the up-GEMM epilogue.
|
||||
@@ -749,17 +790,9 @@ def _fused_moe_kernel_sequence(
|
||||
swiglu_limit=swiglu_limit_for_triton,
|
||||
)
|
||||
elif _is_cuda or _is_hip or _is_xpu:
|
||||
if filter_expert and _is_cuda:
|
||||
# HIP/XPU fall through to the unfiltered path: the down kernel
|
||||
# zeros filtered rows without reading their input.
|
||||
silu_and_mul(
|
||||
intermediate_cache1.view(-1, N),
|
||||
intermediate_cache2,
|
||||
expert_ids=(expert_ids if down_moe_use_tma else topk_ids.view(-1)),
|
||||
expert_step=(config["BLOCK_SIZE_M"] if down_moe_use_tma else 1),
|
||||
)
|
||||
else:
|
||||
silu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2)
|
||||
silu_and_mul(
|
||||
intermediate_cache1.view(-1, N), intermediate_cache2, **row_filter
|
||||
)
|
||||
elif _is_musa:
|
||||
intermediate_cache2 = _silu_and_mul_musa(intermediate_cache1.view(-1, N))
|
||||
else:
|
||||
@@ -790,15 +823,9 @@ def _fused_moe_kernel_sequence(
|
||||
activation_fn = (
|
||||
gelu_tanh_and_mul if activation == "gelu_tanh" else gelu_and_mul
|
||||
)
|
||||
if filter_expert and _is_cuda:
|
||||
activation_fn(
|
||||
intermediate_cache1.view(-1, N),
|
||||
intermediate_cache2,
|
||||
expert_ids=(expert_ids if down_moe_use_tma else topk_ids.view(-1)),
|
||||
expert_step=(config["BLOCK_SIZE_M"] if down_moe_use_tma else 1),
|
||||
)
|
||||
else:
|
||||
activation_fn(intermediate_cache1.view(-1, N), intermediate_cache2)
|
||||
activation_fn(
|
||||
intermediate_cache1.view(-1, N), intermediate_cache2, **row_filter
|
||||
)
|
||||
else:
|
||||
if _has_vllm_ops:
|
||||
getattr(vllm_ops, f"{activation}_and_mul")(
|
||||
@@ -845,6 +872,24 @@ def _fused_moe_kernel_sequence(
|
||||
out_slice = out_hidden_states
|
||||
out_slice.zero_()
|
||||
|
||||
if (
|
||||
_is_cuda
|
||||
and down_moe_use_tma
|
||||
and use_fp8_w8a8
|
||||
and block_shape is not None
|
||||
and a2_scale is None
|
||||
and intermediate_cache2.shape[-1] != block_shape[1]
|
||||
):
|
||||
# Same quant the down call would run, restricted to the rows the down
|
||||
# GEMM reads; a single whole-row group keeps that call's per-token kernel.
|
||||
intermediate_cache2, a2_scale = per_token_group_quant(
|
||||
intermediate_cache2,
|
||||
group_size=block_shape[1],
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
num_tokens_post_padded=num_tokens_post_padded,
|
||||
num_valid_tokens=topk_ids.numel(),
|
||||
)
|
||||
|
||||
invoke_fused_moe_kernel(
|
||||
intermediate_cache2,
|
||||
w2,
|
||||
|
||||
Reference in New Issue
Block a user