From 0c52cb1519d20d9440467caaa61f150e2402437b Mon Sep 17 00:00:00 2001 From: WangHemeng <47976301+wanghemeng@users.noreply.github.com> Date: Mon, 12 Oct 2026 05:10:30 +0200 Subject: [PATCH] [MoE] Skip block-padding rows in the TMA down activation and quant (#42212) --- .../jit/csrc/elementwise/activation.cuh | 89 ++++++++++-- .../jit/csrc/gemm/per_token_group_quant.cuh | 102 ++++++++++++-- .../kernels/ops/activation/activation.py | 100 +++++++++++++- .../ops/quantization/per_token_group_quant.py | 35 ++++- .../moe/moe_runner/triton_utils/fused_moe.py | 85 +++++++++--- .../kernels/ops/activation/test_activation.py | 37 +++++ .../ops/moe/test_moe_tma_down_padding_rows.py | 130 ++++++++++++++++++ .../test_per_token_group_quant.py | 39 ++++++ 8 files changed, 568 insertions(+), 49 deletions(-) create mode 100644 test/registered/kernels/ops/moe/test_moe_tma_down_padding_rows.py diff --git a/python/sglang/kernels/jit/csrc/elementwise/activation.cuh b/python/sglang/kernels/jit/csrc/elementwise/activation.cuh index 1f52a355c3..fa8a1a2365 100644 --- a/python/sglang/kernels/jit/csrc/elementwise/activation.cuh +++ b/python/sglang/kernels/jit/csrc/elementwise/activation.cuh @@ -54,6 +54,12 @@ struct ActivationParams { const int32_t* __restrict__ expert_ids; uint32_t expert_step; float clamp_limit; + // Optional sorted-layout filtering (MoE TMA down path): a row is skipped if it + // is at or past *num_tokens_post_padded or its sorted_token_ids entry is the + // padding sentinel (>= num_valid_tokens). Must match token_mask in fused_moe_kernel. + const int32_t* __restrict__ sorted_token_ids; + const int32_t* __restrict__ num_tokens_post_padded; + uint32_t num_valid_tokens; }; template < @@ -63,7 +69,8 @@ template < bool kFilterExpert, bool kRoundActivation = false, bool kReuseInput = false, - bool kClamp = false> + bool kClamp = false, + bool kFilterSorted = false> __global__ void act_and_mul_kernel(const __grid_constant__ ActivationParams params) { using namespace device; constexpr auto kVecSize = kMaxVecBytes / sizeof(T); @@ -73,7 +80,16 @@ __global__ void act_and_mul_kernel(const __grid_constant__ ActivationParams para const auto token_id = tid / num_vecs; if (token_id >= params.num_tokens) return; - if constexpr (kFilterExpert) { + // Like expert_ids, these are moe_align_block_size outputs the up GEMM already + // waited on (gdc_wait) before triggering this launch, so they may be read before + // PDLWaitPrimary. The host checks that both arrays cover every row, so the three + // loads are issued together instead of one after another. + if constexpr (kFilterSorted) { + const auto num_post = static_cast(*params.num_tokens_post_padded); + const auto sorted_id = static_cast(params.sorted_token_ids[token_id]); + const auto expert_id = params.expert_ids[token_id / params.expert_step]; + if (token_id >= num_post || sorted_id >= params.num_valid_tokens || expert_id == -1) return; + } else if constexpr (kFilterExpert) { if (params.expert_ids[token_id / params.expert_step] == -1) return; } const auto offset = tid % num_vecs; @@ -140,21 +156,26 @@ struct ActivationKernel { using kernel_fn_t = decltype(&act_and_mul_kernel); using unary_kernel_fn_t = decltype(&act_kernel); - template + template < + ActivationKind kAct, + bool kFilterExpert, + bool kRoundActivation = false, + bool kReuseInput = false, + bool kFilterSorted = false> static constexpr kernel_fn_t activation_kernel = - act_and_mul_kernel; + act_and_mul_kernel; static_assert(device::kMaxVecBytes % sizeof(T) == 0, "unsupported data type"); - template + template static kernel_fn_t select_kernel(const std::string& type) { using namespace host; if (type == "silu") { - return activation_kernel; + return activation_kernel; } else if (type == "gelu") { - return activation_kernel; + return activation_kernel; } else if (type == "gelu_tanh") { - return activation_kernel; + return activation_kernel; } else { Panic("unsupported activation type: ", type); } @@ -168,7 +189,10 @@ struct ActivationKernel { const std::string& type, const int32_t* expert_ids, uint32_t expert_step, - float clamp_limit = 0.0f) { + float clamp_limit = 0.0f, + const int32_t* sorted_token_ids = nullptr, + const int32_t* num_tokens_post_padded = nullptr, + uint32_t num_valid_tokens = 0) { using namespace host; auto N = SymbolicSize{"num_tokens"}; @@ -208,12 +232,24 @@ struct ActivationKernel { .expert_ids = expert_ids, .expert_step = expert_step, .clamp_limit = clamp_limit, + .sorted_token_ids = sorted_token_ids, + .num_tokens_post_padded = num_tokens_post_padded, + .num_valid_tokens = num_valid_tokens, }; if constexpr (kClamp) { RuntimeCheck(type == "silu" && expert_ids == nullptr, "clamping requires unfiltered SiLU"); const auto kernel = act_and_mul_kernel; LaunchKernel(num_blocks, kBlockSize, device).enable_pdl(kUsePDL)(kernel, params); + } else if (sorted_token_ids != nullptr) { + // Only the plain (non-rounding, out-of-place) path takes the sorted + // layout; the sorted layout always comes with block-level expert_ids. + if constexpr (kRoundActivation || kReuseInput) { + Panic("sorted filtering is not supported for this activation variant"); + } else { + const auto kernel = select_kernel(type); + LaunchKernel(num_blocks, kBlockSize, device).enable_pdl(kUsePDL)(kernel, params); + } } else if (expert_ids != nullptr) { RuntimeCheck(expert_step > 0, "expert_step must be positive"); const auto kernel = select_kernel(type); @@ -254,6 +290,41 @@ struct ActivationKernel { launch(input, out, type, static_cast(expert_ids.data_ptr()), static_cast(expert_step)); } + /// \brief Gated activation over the sorted, block-padded MoE layout. + /// \param expert_ids Per-block expert ids (-1 = non-local expert), one per expert_step rows. + /// \param sorted_token_ids moe_align_block_size output; entries >= num_valid_tokens are padding. + /// \param num_tokens_post_padded Device scalar; rows at or past it are padding. + /// Rows the MoE GEMM never reads are skipped and left unwritten in out. + static void run_activation_sorted_filtered( + const tvm::ffi::TensorView input, + const tvm::ffi::TensorView out, + const tvm::ffi::TensorView expert_ids, + int64_t expert_step, + const tvm::ffi::TensorView sorted_token_ids, + const tvm::ffi::TensorView num_tokens_post_padded, + int64_t num_valid_tokens, + std::string type) { + using namespace host; + CHECK_HOST(is_type(expert_ids.dtype())) << "expert_ids must have dtype int32"; + CHECK_HOST(is_type(sorted_token_ids.dtype())) << "sorted_token_ids must have dtype int32"; + CHECK_HOST(is_type(num_tokens_post_padded.dtype())) << "num_tokens_post_padded must have dtype int32"; + CHECK_HOST(expert_step >= 1) << "expert_step must be positive, got " << expert_step; + CHECK_HOST(sorted_token_ids.numel() >= input.size(0)) << "sorted_token_ids must cover every row"; + CHECK_HOST(expert_ids.numel() * expert_step >= input.size(0)) << "expert_ids must cover every row"; + CHECK_HOST(0 <= num_valid_tokens && num_valid_tokens <= std::numeric_limits::max()) + << "num_valid_tokens out of range: " << num_valid_tokens; + launch( + input, + out, + type, + static_cast(expert_ids.data_ptr()), + static_cast(expert_step), + /*clamp_limit=*/0.0f, + static_cast(sorted_token_ids.data_ptr()), + static_cast(num_tokens_post_padded.data_ptr()), + static_cast(num_valid_tokens)); + } + template static constexpr auto unary_kernel = act_kernel; diff --git a/python/sglang/kernels/jit/csrc/gemm/per_token_group_quant.cuh b/python/sglang/kernels/jit/csrc/gemm/per_token_group_quant.cuh index f4c751d6be..23cd4407fa 100644 --- a/python/sglang/kernels/jit/csrc/gemm/per_token_group_quant.cuh +++ b/python/sglang/kernels/jit/csrc/gemm/per_token_group_quant.cuh @@ -15,6 +15,8 @@ #include #include #include +#include +#include namespace sglang { @@ -209,6 +211,29 @@ struct MaskedQuantKernelParams { uint32_t masked_m_stride; // 1 (int32) or 2 (int64) }; +struct SortedQuantKernelParams { + QuantKernelParams base; + // Sorted, block-padded MoE layout (moe_align_block_size output): a row at or + // past *num_tokens_post_padded, or holding the padding sentinel + // (>= num_valid_tokens), is never read by the MoE GEMM. Must match token_mask + // in fused_moe_kernel. + const int32_t* __restrict__ sorted_token_ids; + const int32_t* __restrict__ num_tokens_post_padded; + uint32_t num_valid_tokens; +}; + +namespace detail { + +SGL_DEVICE const QuantKernelParams& base_params(const QuantKernelParams& params) { + return params; +} + +SGL_DEVICE const QuantKernelParams& base_params(const SortedQuantKernelParams& params) { + return params.base; +} + +} // namespace detail + // PDL is a launch-scheduling knob, not a quant property, so it is a separate // template parameter of the kernels/launchers rather than part of QuantTrait. template < @@ -392,12 +417,14 @@ struct QuantTrait { }; // --------------------------------------------------------------------------- -// Flat schedule: one subwarp (kNumLanes) per group over a linear grid. +// Flat schedule: one subwarp (kNumLanes) per group over a linear grid. With +// kSortedRows, rows are in the sorted MoE layout and padding rows are skipped. // --------------------------------------------------------------------------- -template +template __global__ __launch_bounds__(Trait::kBlockSize) void per_token_group_quant_flat_kernel( - const __grid_constant__ QuantKernelParams params) { + const __grid_constant__ std::conditional_t kernel_params) { using namespace device; + const QuantKernelParams& params = detail::base_params(kernel_params); constexpr uint32_t kNumLanes = Trait::kNumLanes; constexpr uint32_t kWorkPerWarp = kWarpThreads / kNumLanes; const auto num_groups = params.scale.num_groups; @@ -412,6 +439,16 @@ __global__ __launch_bounds__(Trait::kBlockSize) void per_token_group_quant_flat_ const auto token_idx = work_id / num_groups; const auto group_idx = work_id % num_groups; PDLWaitPrimary(); + if constexpr (kSortedRows) { + // run() reduces across the whole warp, so only warps whose rows are all + // padding exit; a mixed warp also quantizes its padding rows (never read). + // The host checks sorted_token_ids covers every row, so both loads issue together. + const auto& sorted = kernel_params; + const auto num_post = static_cast(*sorted.num_tokens_post_padded); + const auto sorted_id = static_cast(sorted.sorted_token_ids[token_idx]); + const bool skip = token_idx >= num_post || sorted_id >= sorted.num_valid_tokens; + if (__all_sync(warp::kFullMask, skip)) return; + } Trait::run(params, 0, token_idx, group_idx, lane_id); PDLTriggerSecondary(); } @@ -562,6 +599,17 @@ QuantHostContext build_quant_context( // }; } +template +void launch_flat_kernel(const QuantHostContext& ctx, const Params& params) { + using namespace host; + const auto& p = ctx.params; + const int64_t total_threads = int64_t{p.num_tokens} * p.scale.num_groups * Trait::kNumLanes; + if (total_threads == 0) return; + const uint32_t num_blocks = div_ceil(total_threads, int64_t{Trait::kBlockSize}); + LaunchKernel(num_blocks, Trait::kBlockSize, ctx.device) + .config({.use_pdl = kUsePDL})(per_token_group_quant_flat_kernel, params); +} + template < typename InputType, typename QuantType, @@ -575,14 +623,48 @@ struct PerTokenGroupQuantFlatKernel { using Trait = QuantTrait; static void run(tvm::ffi::TensorView input, tvm::ffi::TensorView output_q, tvm::ffi::TensorView output_s) { - using namespace host; const auto ctx = build_quant_context(input, output_q, output_s); - const auto& p = ctx.params; - const int64_t total_threads = int64_t{p.num_tokens} * p.scale.num_groups * Trait::kNumLanes; - if (total_threads == 0) return; - const uint32_t num_blocks = div_ceil(total_threads, int64_t{Trait::kBlockSize}); - LaunchKernel(num_blocks, Trait::kBlockSize, ctx.device) - .config({.use_pdl = kUsePDL})(per_token_group_quant_flat_kernel, p); + launch_flat_kernel(ctx, ctx.params); + } +}; + +template < + typename InputType, + typename QuantType, + uint32_t kGroupSize, + bool kUe8m0, + bool kRowMajor, + bool kAligned, + bool kFuseSiluAndMul, + bool kUsePDL> +struct PerTokenGroupQuantSortedKernel { + using Trait = QuantTrait; + + /// \brief Flat quantization over the sorted, block-padded MoE layout. + /// \param sorted_token_ids moe_align_block_size output; entries >= num_valid_tokens are padding. + /// \param num_tokens_post_padded Device scalar; rows at or past it are padding. + /// Rows the MoE GEMM never reads may be skipped and left unwritten. + static void + run(tvm::ffi::TensorView input, + tvm::ffi::TensorView output_q, + tvm::ffi::TensorView output_s, + tvm::ffi::TensorView sorted_token_ids, + tvm::ffi::TensorView num_tokens_post_padded, + int64_t num_valid_tokens) { + using namespace host; + CHECK_HOST(is_type(sorted_token_ids.dtype())) << "sorted_token_ids must have dtype int32"; + CHECK_HOST(is_type(num_tokens_post_padded.dtype())) << "num_tokens_post_padded must have dtype int32"; + CHECK_HOST(0 <= num_valid_tokens && num_valid_tokens <= std::numeric_limits::max()) + << "num_valid_tokens out of range: " << num_valid_tokens; + const auto ctx = build_quant_context(input, output_q, output_s); + CHECK_HOST(sorted_token_ids.numel() >= ctx.params.num_tokens) << "sorted_token_ids must cover every row"; + const auto params = SortedQuantKernelParams{ + ctx.params, + static_cast(sorted_token_ids.data_ptr()), + static_cast(num_tokens_post_padded.data_ptr()), + static_cast(num_valid_tokens), + }; + launch_flat_kernel(ctx, params); } }; diff --git a/python/sglang/kernels/ops/activation/activation.py b/python/sglang/kernels/ops/activation/activation.py index 1b4fdf956c..4ee278ce07 100644 --- a/python/sglang/kernels/ops/activation/activation.py +++ b/python/sglang/kernels/ops/activation/activation.py @@ -42,6 +42,10 @@ def activation_module(dtype: torch.dtype, *, fast_math: bool = True) -> Module: "run_activation_filtered", f"ActivationKernel<{args}>::run_activation_filtered", ), + ( + "run_activation_sorted_filtered", + f"ActivationKernel<{args}>::run_activation_sorted_filtered", + ), ( "run_unary_activation", f"ActivationKernel<{args}>::run_unary_activation", @@ -102,12 +106,41 @@ def _run_activation_filtered_inplace( module.run_activation_filtered(input_2d, out_2d, expert_ids, expert_step, op_name) +@register_custom_op(mutates_args=["out"]) +def _run_activation_sorted_filtered_inplace( + op_name: str, + input: torch.Tensor, + out: torch.Tensor, + expert_ids: torch.Tensor, + expert_step: int, + sorted_token_ids: torch.Tensor, + num_tokens_post_padded: torch.Tensor, + num_valid_tokens: int, +) -> None: + hidden_size = input.shape[-1] // 2 + module = activation_module(input.dtype) + module.run_activation_sorted_filtered( + input.view(-1, hidden_size * 2), + out.view(-1, hidden_size), + expert_ids, + expert_step, + sorted_token_ids, + num_tokens_post_padded, + num_valid_tokens, + op_name, + ) + + def run_activation( op_name: str, input: torch.Tensor, out: Optional[torch.Tensor], expert_ids: Optional[torch.Tensor] = None, expert_step: int = 1, + *, + sorted_token_ids: Optional[torch.Tensor] = None, + num_tokens_post_padded: Optional[torch.Tensor] = None, + num_valid_tokens: int = 0, ) -> torch.Tensor: """Apply ``op_name`` activation followed by element-wise multiplication. @@ -115,6 +148,12 @@ def run_activation( routed expert id is ``-1``. ``expert_step`` is 1 for per-token routing and ``BLOCK_SIZE_M`` for sorted/TMA routing — i.e. ``expert_ids[token_id // expert_step]`` is consulted before computing each row. + + For the sorted, block-padded layout (``moe_align_block_size`` output with + ``expert_step = BLOCK_SIZE_M``), also pass ``sorted_token_ids``, + ``num_tokens_post_padded`` and ``num_valid_tokens`` (``topk_ids.numel()``) + to skip the padding rows the down GEMM never reads; skipped rows of ``out`` + are left unwritten. """ assert op_name in SUPPORTED_ACTIVATIONS, f"Unsupported activation: {op_name}" hidden_size = input.shape[-1] // 2 @@ -128,7 +167,21 @@ def run_activation( # true dtype, so normalize here instead of asserting downstream. if expert_ids.dtype != torch.int32: expert_ids = expert_ids.to(torch.int32) - _run_activation_filtered_inplace(op_name, input, out, expert_ids, expert_step) + if sorted_token_ids is None: + _run_activation_filtered_inplace( + op_name, input, out, expert_ids, expert_step + ) + else: + _run_activation_sorted_filtered_inplace( + op_name, + input, + out, + expert_ids, + expert_step, + sorted_token_ids, + num_tokens_post_padded, + num_valid_tokens, + ) return out @@ -177,8 +230,21 @@ def silu_and_mul( out: Optional[torch.Tensor] = None, expert_ids: Optional[torch.Tensor] = None, expert_step: int = 1, + *, + sorted_token_ids: Optional[torch.Tensor] = None, + num_tokens_post_padded: Optional[torch.Tensor] = None, + num_valid_tokens: int = 0, ) -> torch.Tensor: - return run_activation("silu", input, out, expert_ids, expert_step) + return run_activation( + "silu", + input, + out, + expert_ids, + expert_step, + sorted_token_ids=sorted_token_ids, + num_tokens_post_padded=num_tokens_post_padded, + num_valid_tokens=num_valid_tokens, + ) def silu_and_mul_with_activation_rounding( @@ -216,8 +282,21 @@ def gelu_and_mul( out: Optional[torch.Tensor] = None, expert_ids: Optional[torch.Tensor] = None, expert_step: int = 1, + *, + sorted_token_ids: Optional[torch.Tensor] = None, + num_tokens_post_padded: Optional[torch.Tensor] = None, + num_valid_tokens: int = 0, ) -> torch.Tensor: - return run_activation("gelu", input, out, expert_ids, expert_step) + return run_activation( + "gelu", + input, + out, + expert_ids, + expert_step, + sorted_token_ids=sorted_token_ids, + num_tokens_post_padded=num_tokens_post_padded, + num_valid_tokens=num_valid_tokens, + ) def gelu_tanh_and_mul( @@ -225,5 +304,18 @@ def gelu_tanh_and_mul( out: Optional[torch.Tensor] = None, expert_ids: Optional[torch.Tensor] = None, expert_step: int = 1, + *, + sorted_token_ids: Optional[torch.Tensor] = None, + num_tokens_post_padded: Optional[torch.Tensor] = None, + num_valid_tokens: int = 0, ) -> torch.Tensor: - return run_activation("gelu_tanh", input, out, expert_ids, expert_step) + return run_activation( + "gelu_tanh", + input, + out, + expert_ids, + expert_step, + sorted_token_ids=sorted_token_ids, + num_tokens_post_padded=num_tokens_post_padded, + num_valid_tokens=num_valid_tokens, + ) diff --git a/python/sglang/kernels/ops/quantization/per_token_group_quant.py b/python/sglang/kernels/ops/quantization/per_token_group_quant.py index ca608e4251..fcb2133e2d 100644 --- a/python/sglang/kernels/ops/quantization/per_token_group_quant.py +++ b/python/sglang/kernels/ops/quantization/per_token_group_quant.py @@ -32,6 +32,7 @@ def _jit_module( fuse_silu_and_mul: bool, masked_layout: bool, use_pdl: bool, + sorted_layout: bool = False, ) -> Module: assert in_dtype in _SUPPORTED_INPUT_DTYPES assert out_dtype in _SUPPORTED_OUTPUT_DTYPES @@ -46,15 +47,13 @@ def _jit_module( fuse_silu_and_mul, use_pdl, ) - launcher = ( - "PerTokenGroupQuantMaskedKernel" - if masked_layout - else "PerTokenGroupQuantFlatKernel" - ) + assert not (masked_layout and sorted_layout) + layout = "masked" if masked_layout else ("sorted" if sorted_layout else "flat") + launcher = f"PerTokenGroupQuant{layout.capitalize()}Kernel" return load_jit( "per_token_group_quant", *trait_args, - "masked" if masked_layout else "flat", + layout, cuda_files=["gemm/per_token_group_quant.cuh"], cuda_wrappers=[("per_token_group_quant", f"{launcher}<{trait_args}>::run")], extra_cuda_cflags=["--use_fast_math"], @@ -95,6 +94,9 @@ def _per_token_group_quant_custom_op( fuse_silu_and_mul: bool = False, masked_m: Optional[torch.Tensor] = None, expected_m: Optional[int] = None, + sorted_token_ids: Optional[torch.Tensor] = None, + num_tokens_post_padded: Optional[torch.Tensor] = None, + num_valid_tokens: int = 0, ) -> None: num_groups = output_q.shape[-1] // group_size row_major, aligned = _infer_scale_layout(output_s, scale_ue8m0, num_groups) @@ -108,11 +110,21 @@ def _per_token_group_quant_custom_op( bool(fuse_silu_and_mul), masked_m is not None, is_arch_support_pdl(), + sorted_token_ids is not None, ) if masked_m is not None: module.per_token_group_quant( input, output_q, output_s, masked_m, int(expected_m or -1) ) + elif sorted_token_ids is not None: + module.per_token_group_quant( + input, + output_q, + output_s, + sorted_token_ids, + num_tokens_post_padded, + num_valid_tokens, + ) else: module.per_token_group_quant(input, output_q, output_s) @@ -171,6 +183,9 @@ def per_token_group_quant( *, out_dtype: Optional[torch.dtype] = None, column_major_scales: bool = False, + sorted_token_ids: Optional[torch.Tensor] = None, + num_tokens_post_padded: Optional[torch.Tensor] = None, + num_valid_tokens: int = 0, ) -> Tuple[torch.Tensor, torch.Tensor]: """Per-token-group quantization. Returns ``(output_q, output_s)``. @@ -194,6 +209,11 @@ def per_token_group_quant( ``expected_m`` (masked only) is an optional expected-tokens-per-expert hint. + ``sorted_token_ids`` selects the sorted layout: ``input`` rows follow + ``moe_align_block_size`` (with ``num_tokens_post_padded`` and + ``num_valid_tokens = topk_ids.numel()``), and rows the MoE GEMM never reads + may be skipped and left unwritten. + Inputs are bf16/fp16; group size is one of 16/32/64/128/256; the quant range follows ``output_q.dtype`` (fp8_e4m3: +-448, int8: [-128, 127]). """ @@ -219,5 +239,8 @@ def per_token_group_quant( fuse_silu_and_mul=fuse_silu_and_mul, masked_m=masked_m, expected_m=expected_m, + sorted_token_ids=sorted_token_ids, + num_tokens_post_padded=num_tokens_post_padded, + num_valid_tokens=num_valid_tokens, ) return output_q, output_s diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py index d4baddfe68..ad6b8c9d77 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py @@ -64,6 +64,9 @@ if _is_cuda: silu_and_mul, silu_and_mul_with_activation_rounding, ) + from sglang.kernels.ops.quantization.per_token_group_quant import ( + per_token_group_quant, + ) elif _is_cpu and _is_cpu_amx_available: pass elif _is_hip: @@ -125,6 +128,34 @@ def _validate_fused_swiglu_interleaved( ) +def _activation_row_filter( + *, + down_moe_use_tma: bool, + filter_expert: bool, + topk_ids: torch.Tensor, + expert_ids: torch.Tensor, + sorted_token_ids: torch.Tensor, + num_tokens_post_padded: torch.Tensor, + block_size_m: int, +) -> Dict[str, Any]: + """Row filter for the CUDA JIT gated activations; empty computes every row.""" + # HIP/XPU compute every row: the down kernel zeros filtered rows without + # reading their input. + if not _is_cuda or not (down_moe_use_tma or filter_expert): + return {} + if not down_moe_use_tma: + return dict(expert_ids=topk_ids.view(-1), expert_step=1) + # The TMA down buffer is sorted, block-padded and sized for the worst case; + # compute only the rows the down GEMM reads. + return dict( + expert_ids=expert_ids, + expert_step=block_size_m, + sorted_token_ids=sorted_token_ids, + num_tokens_post_padded=num_tokens_post_padded, + num_valid_tokens=topk_ids.numel(), + ) + + def _use_moe_sum_reduce_torch_compile(num_tokens: int) -> bool: return num_tokens <= 32 and not is_batch_invariant_mode_enabled() @@ -670,6 +701,16 @@ def _fused_moe_kernel_sequence( dtype=hidden_states.dtype, ) + row_filter = _activation_row_filter( + down_moe_use_tma=down_moe_use_tma, + filter_expert=filter_expert, + topk_ids=topk_ids, + expert_ids=expert_ids, + sorted_token_ids=sorted_token_ids, + num_tokens_post_padded=num_tokens_post_padded, + block_size_m=config["BLOCK_SIZE_M"], + ) + # Activation function with multiplication if fuse_swiglu_interleaved: # silu(gate) * up was already applied by the up-GEMM epilogue. @@ -749,17 +790,9 @@ def _fused_moe_kernel_sequence( swiglu_limit=swiglu_limit_for_triton, ) elif _is_cuda or _is_hip or _is_xpu: - if filter_expert and _is_cuda: - # HIP/XPU fall through to the unfiltered path: the down kernel - # zeros filtered rows without reading their input. - silu_and_mul( - intermediate_cache1.view(-1, N), - intermediate_cache2, - expert_ids=(expert_ids if down_moe_use_tma else topk_ids.view(-1)), - expert_step=(config["BLOCK_SIZE_M"] if down_moe_use_tma else 1), - ) - else: - silu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2) + silu_and_mul( + intermediate_cache1.view(-1, N), intermediate_cache2, **row_filter + ) elif _is_musa: intermediate_cache2 = _silu_and_mul_musa(intermediate_cache1.view(-1, N)) else: @@ -790,15 +823,9 @@ def _fused_moe_kernel_sequence( activation_fn = ( gelu_tanh_and_mul if activation == "gelu_tanh" else gelu_and_mul ) - if filter_expert and _is_cuda: - activation_fn( - intermediate_cache1.view(-1, N), - intermediate_cache2, - expert_ids=(expert_ids if down_moe_use_tma else topk_ids.view(-1)), - expert_step=(config["BLOCK_SIZE_M"] if down_moe_use_tma else 1), - ) - else: - activation_fn(intermediate_cache1.view(-1, N), intermediate_cache2) + activation_fn( + intermediate_cache1.view(-1, N), intermediate_cache2, **row_filter + ) else: if _has_vllm_ops: getattr(vllm_ops, f"{activation}_and_mul")( @@ -845,6 +872,24 @@ def _fused_moe_kernel_sequence( out_slice = out_hidden_states out_slice.zero_() + if ( + _is_cuda + and down_moe_use_tma + and use_fp8_w8a8 + and block_shape is not None + and a2_scale is None + and intermediate_cache2.shape[-1] != block_shape[1] + ): + # Same quant the down call would run, restricted to the rows the down + # GEMM reads; a single whole-row group keeps that call's per-token kernel. + intermediate_cache2, a2_scale = per_token_group_quant( + intermediate_cache2, + group_size=block_shape[1], + sorted_token_ids=sorted_token_ids, + num_tokens_post_padded=num_tokens_post_padded, + num_valid_tokens=topk_ids.numel(), + ) + invoke_fused_moe_kernel( intermediate_cache2, w2, diff --git a/test/registered/kernels/ops/activation/test_activation.py b/test/registered/kernels/ops/activation/test_activation.py index 1a5facdd34..a4e8589472 100644 --- a/test/registered/kernels/ops/activation/test_activation.py +++ b/test/registered/kernels/ops/activation/test_activation.py @@ -168,6 +168,43 @@ def test_activation_filter_expert_none_skipped(op_name: str) -> None: torch.testing.assert_close(out_filtered, out_unfiltered, atol=0.0, rtol=0.0) +@pytest.mark.parametrize("op_name", OPS) +@pytest.mark.parametrize("block_m", [16, 64]) +def test_activation_sorted_rows(op_name: str, block_m: int) -> None: + """Sorted MoE layout: rows past num_tokens_post_padded, padding sentinels and + -1 expert blocks are left untouched; every other row is bit-identical.""" + num_valid, num_blocks = 300, 40 + rows = num_blocks * block_m + 7 # buffer tail past the last full block + num_post = (num_blocks - 3) * block_m + sorted_ids = torch.randint(0, num_valid, (rows,), dtype=torch.int32, device="cuda") + sorted_ids[torch.rand(rows, device="cuda") < 0.3] = num_valid + expert_ids = torch.randint( + 0, 8, (num_blocks + 1,), dtype=torch.int32, device="cuda" + ) + expert_ids[::5] = -1 + x = torch.randn(rows, 1024, dtype=torch.bfloat16, device="cuda") + out = torch.full((rows, 512), float("nan"), dtype=x.dtype, device="cuda") + + run_activation( + op_name, + x, + out, + expert_ids, + block_m, + sorted_token_ids=sorted_ids, + num_tokens_post_padded=torch.tensor( + [num_post], dtype=torch.int32, device="cuda" + ), + num_valid_tokens=num_valid, + ) + + row = torch.arange(rows, device="cuda") + kept = (row < num_post) & (sorted_ids < num_valid) + kept &= expert_ids[row // block_m] != -1 + assert torch.equal(out[kept], run_activation(op_name, x, None)[kept]) + assert torch.isnan(out[~kept]).all() + + def test_activation_filter_expert_int64_under_torch_compile() -> None: """torch.topk routing ids remain usable after AOTAutograd realizes int64.""" shape = (32, 512) diff --git a/test/registered/kernels/ops/moe/test_moe_tma_down_padding_rows.py b/test/registered/kernels/ops/moe/test_moe_tma_down_padding_rows.py new file mode 100644 index 0000000000..ee5503e2f4 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_moe_tma_down_padding_rows.py @@ -0,0 +1,130 @@ +"""MoE TMA down path must stay bit-identical to the non-TMA path. + +The TMA down path keeps the activation and the down-input quant in the sorted, +block-padded buffer sized for the worst case and skips the padding rows the down +GEMM never reads. Skipping must not change any result, so the whole layer is +compared bitwise; a tolerance would hide a row mix-up. +""" + +import unittest + +import torch + +from sglang.srt.runtime_context import get_context, get_parallel +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +E, TOPK, HIDDEN, INTER, BLOCK = 256, 8, 2048, 512, [128, 128] +CASES = ((1, 16), (32, 16), (300, 32), (2048, 64)) # (num_tokens, BLOCK_SIZE_M) + + +@unittest.skipUnless( + torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 9, + "requires sm90+ for the TMA down path", +) +class TestMoeTmaDownPaddingRows(CustomTestCase): + @classmethod + def setUpClass(cls): + torch.manual_seed(0) + cls.weights = { + False: ( + (torch.randn(E, 2 * INTER, HIDDEN, device="cuda") / HIDDEN**0.5).to( + torch.bfloat16 + ), + (torch.randn(E, HIDDEN, INTER, device="cuda") / INTER**0.5).to( + torch.bfloat16 + ), + None, + None, + ), + True: ( + torch.randn(E, 2 * INTER, HIDDEN, device="cuda").to( + torch.float8_e4m3fn + ), + torch.randn(E, HIDDEN, INTER, device="cuda").to(torch.float8_e4m3fn), + torch.rand(E, 2 * INTER // 128, HIDDEN // 128, device="cuda") / 32 + + 0.01, + torch.rand(E, HIDDEN // 128, INTER // 128, device="cuda") / 16 + 0.02, + ), + } + + def _layer(self, m, block_m, down_tma, fp8): + from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import ( + _fused_moe_kernel_sequence, + ) + from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import ( + moe_align_block_size, + ) + + torch.manual_seed(m) + hidden = torch.randn(m, HIDDEN, device="cuda", dtype=torch.bfloat16) + topk_ids = torch.rand(m, E, device="cuda").topk(TOPK, dim=-1).indices + topk_ids = topk_ids.to(torch.int32) + topk_weights = torch.rand(m, TOPK, device="cuda").softmax(-1) + sorted_ids, expert_ids, num_post = moe_align_block_size(topk_ids, block_m, E) + config = dict( + BLOCK_SIZE_M=block_m, + BLOCK_SIZE_N=64, + BLOCK_SIZE_K=128, + GROUP_SIZE_M=1, + num_warps=4, + num_stages=3, + ) + w1, w2, w1_s, w2_s = self.weights[fp8] + return _fused_moe_kernel_sequence( + hidden, + w1, + w2, + topk_weights, + topk_ids, + sorted_ids, + expert_ids, + num_post, + config, + dict(config, BLOCK_SIZE_N=128), + down_tma, + False, + b1=None, + b2=None, + use_fp8_w8a8=fp8, + use_int8_w8a8=False, + use_int8_w8a16=False, + use_int4_w4a16=False, + per_channel_quant=False, + w1_scale=w1_s, + w2_scale=w2_s, + w1_zp=None, + w2_zp=None, + a1_scale=None, + a2_scale=None, + block_shape=BLOCK if fp8 else None, + activation="silu", + is_gated=True, + no_combine=False, + inplace=False, + apply_router_weight_on_input=False, + routed_scaling_factor=None, + gemm1_alpha=None, + gemm1_limit=None, + filter_expert=False, + ) + + def test_tma_down_layer_bitwise_matches_non_tma(self): + with ( + get_context().override_server_args(enable_deterministic_inference=False), + get_parallel().override(tp_group=None), + ): + for fp8 in (True, False): + for m, block_m in CASES: + with self.subTest(fp8=fp8, m=m, block_m=block_m): + tma = self._layer(m, block_m, down_tma=True, fp8=fp8) + ref = self._layer(m, block_m, down_tma=False, fp8=fp8) + self.assertFalse(tma.isnan().any()) + self.assertGreater(ref.abs().max().item(), 0.1) + self.assertTrue(torch.equal(tma, ref)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/quantization/test_per_token_group_quant.py b/test/registered/kernels/ops/quantization/test_per_token_group_quant.py index 642bbb2737..3554e8aeef 100644 --- a/test/registered/kernels/ops/quantization/test_per_token_group_quant.py +++ b/test/registered/kernels/ops/quantization/test_per_token_group_quant.py @@ -551,6 +551,45 @@ def test_masked_fused(): assert torch.all(x_q[e, m:].view(torch.int8) == 0), "padding touched" +@pytest.mark.parametrize("column_major", [False, True]) +def test_sorted_rows(column_major): + """Sorted MoE layout (TMA down input): rows the MoE GEMM never reads are + skipped, every other row is bit-identical to the flat kernel. hidden=512 is + one warp per row (4 groups), so each padding row is skipped as a whole.""" + torch.manual_seed(11) + num_valid, block_m, num_blocks, hidden = 300, 16, 40, 512 + rows = num_blocks * block_m + 7 + num_post = (num_blocks - 3) * block_m + sorted_ids = torch.randint(0, num_valid, (rows,), dtype=torch.int32, device="cuda") + sorted_ids[torch.rand(rows, device="cuda") < 0.3] = num_valid + x = torch.randn(rows, hidden, device="cuda", dtype=torch.bfloat16) + + ref_q = torch.zeros(rows, hidden, device="cuda", dtype=fp8_dtype) + ref_s = _alloc_scale((rows, hidden), column_major=column_major, scale_ue8m0=False) + per_token_group_quant(x, ref_q, ref_s, G) + x_q = torch.full_like(ref_q.view(torch.uint8), 0x7F).view(fp8_dtype) + x_s = _alloc_scale((rows, hidden), column_major=column_major, scale_ue8m0=False) + x_s.fill_(float("nan")) + per_token_group_quant( + x, + x_q, + x_s, + G, + sorted_token_ids=sorted_ids, + num_tokens_post_padded=torch.tensor( + [num_post], dtype=torch.int32, device="cuda" + ), + num_valid_tokens=num_valid, + ) + + row = torch.arange(rows, device="cuda") + kept = (row < num_post) & (sorted_ids < num_valid) + assert torch.equal(x_q.view(torch.uint8)[kept], ref_q.view(torch.uint8)[kept]) + assert torch.equal(x_s[kept], ref_s[kept]) + assert (x_q.view(torch.uint8)[~kept] == 0x7F).all(), "padding row written" + assert torch.isnan(x_s[~kept]).all(), "padding scale written" + + NON_FINITE_CASES = get_ci_test_range( list( itertools.product(