[kernel] SwiGLU-OAI MXFP8 producer; move swizzled MXFP8 producers to ops/quantization (#42810)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Denji-kk <51020465+Denji-kk@users.noreply.github.com>
This commit is contained in:
Chunan Zeng
2026-10-10 08:51:57 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 Denji-kk
parent 961bcf481d
commit 6acaf3374a
19 changed files with 375 additions and 133 deletions
@@ -48,7 +48,7 @@ import triton # type: ignore
import triton.language as tl # type: ignore
from sglang.kernels.jit.utils import get_jit_cuda_arch
from sglang.kernels.ops.diffusion.common.numerics import (
from sglang.kernels.numerics import (
cuda_rsqrtf,
div_rn_f32,
round_bf16_to_fp32,
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
"""Numerical primitives shared by bit-exact diffusion Triton kernels."""
"""Numerical primitives shared by bit-exact Triton kernels."""
import triton # type: ignore
import triton.language as tl # type: ignore
@@ -37,11 +37,11 @@ norm/ RMSNorm / LayerNorm / GroupNorm and their fused epilogues
modulate/ adaLN modulate, gating, timestep conditioning
rope/ rotary embeddings and the QK-norm chains fused into them
activation/ SiLU / GLU / GELU fusions
quantization/ MXFP8 producers whose scales land in the GEMM's swizzled layout
quantization/ FP8 rowwise producers fused into the GEMM input path
attention/ sparse linear attention, gated delta-net
routing/ diffusion-model MoE routing and expert selection
layout/ pure data movement: USP/Ulysses relayout, varlen pack, causal pad
common/ numerics primitives, platform predicates, non-Triton fallbacks
common/ platform predicates, non-Triton fallbacks (numerics: kernels/numerics.py)
sites/ request-scoped mount policy — NOT kernels (see below)
ext/ JIT C++/CUDA extensions (Hunyuan3D raster/inpaint) — NOT kernels
../../kda_kernels/ agent-generated implementations and their JIT CUDA sources
@@ -192,6 +192,8 @@ tensor copy at each residual site.
### MXFP8 producers (online `mxfp8`, cuBLASLt block-scaled GEMM on SM100)
Defined in `sglang.kernels.ops.quantization.mxfp8_swizzled_triton` and re-exported here.
| Entry point | Backend | Contract |
|---|---|---|
| `mxfp8_quantize_swizzled` | Triton | bit-exact vs `flashinfer.mxfp8_quantize(x, True)`: e4m3 payload + block-32 E8M0 scales in the `SWIZZLE_32_4_4` layout; weights at load and any bf16 GEMM input |
@@ -13,7 +13,7 @@ Layout -- ordinary implementations use one subpackage per **operator domain**
``layout``).
Implementations generated by kernel-design agents live in
``sglang.kernels.kda_kernels`` and are still exported through this facade.
``common`` holds shared numerics and platform plumbing, ``sites`` the
``common`` holds platform plumbing, ``sites`` the
request-scoped mount policy, and ``ext`` JIT C++/CUDA extensions that are not
kernels. Start from ``README.md``: several norms look interchangeable and are
not.
@@ -486,21 +486,21 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
(
"diffusion.mxfp8_quantize_swizzled",
KernelBackend.TRITON,
"quantization.mxfp8_swizzled_triton:mxfp8_quantize_swizzled",
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:mxfp8_quantize_swizzled",
_CUDA,
"bf16 -> MXFP8 (e4m3, block-32 E8M0 scales in the cuBLASLt swizzled layout).",
),
(
"diffusion.silu_mul_mxfp8",
KernelBackend.TRITON,
"quantization.mxfp8_swizzled_triton:silu_mul_mxfp8",
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:silu_mul_mxfp8",
_CUDA,
"SwiGLU + MXFP8 quant for the online mxfp8 fc2 input.",
),
(
"diffusion.indexed_scale_shift_mxfp8_",
KernelBackend.TRITON,
"quantization.mxfp8_swizzled_triton:indexed_scale_shift_mxfp8_",
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:indexed_scale_shift_mxfp8_",
_CUDA,
"Indexed adaLN modulation + MXFP8 quant for the online mxfp8 qkv/fc1 inputs.",
),
@@ -745,11 +745,11 @@ _EXPORTS: dict[str, str] = {
"vdn_frame_stats_prep": "attention.vdn_linear_branch_triton",
"vdn_gather_linear_state": "attention.vdn_linear_branch_triton",
"vdn_linear_epilogue": "attention.vdn_linear_branch_triton",
"can_use_mxfp8_swizzled": "quantization.mxfp8_swizzled_triton",
"can_use_silu_mul_mxfp8": "quantization.mxfp8_swizzled_triton",
"indexed_scale_shift_mxfp8_": "quantization.mxfp8_swizzled_triton",
"mxfp8_quantize_swizzled": "quantization.mxfp8_swizzled_triton",
"silu_mul_mxfp8": "quantization.mxfp8_swizzled_triton",
"can_use_mxfp8_swizzled": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
"can_use_silu_mul_mxfp8": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
"indexed_scale_shift_mxfp8_": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
"mxfp8_quantize_swizzled": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
"silu_mul_mxfp8": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
"can_use_usp_merge_heads": "layout.usp_relayout_jit",
"usp_merge_heads": "layout.usp_relayout_jit",
"build_inv_indices": "layout.varlen_pack_pad_triton",
@@ -7,7 +7,7 @@ import torch
import triton # type: ignore
import triton.language as tl # type: ignore
from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32
from sglang.kernels.numerics import round_bf16_to_fp32
@triton.jit
@@ -20,7 +20,7 @@ import torch
import triton # type: ignore
import triton.language as tl # type: ignore
from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32
from sglang.kernels.numerics import round_bf16_to_fp32
from sglang.srt.utils.custom_op import register_custom_op
@@ -1,6 +1,5 @@
"""Shared infrastructure for the diffusion kernels -- no kernels of its own.
- ``numerics`` : rounding/opmath primitives the bit-exact kernels are built from
- ``platform`` : device predicates and the Triton-vs-fallback selector
- ``fallback_*``: pure-torch / NPU / MPS implementations for Triton-less devices
"""
@@ -4,7 +4,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32
from sglang.kernels.numerics import round_bf16_to_fp32
# Column tiles per row. One program per whole row padded to a power of two
# masked off 37.5% of MiniMax-H3's 5376-wide rows; 2048-wide tiles run both
@@ -2,10 +2,10 @@ import torch
import triton # type: ignore
import triton.language as tl # type: ignore
from sglang.kernels.numerics import mul_rn_f32
from sglang.kernels.ops.diffusion.common.fallback_torch import (
fuse_scale_shift_kernel_native,
)
from sglang.kernels.ops.diffusion.common.numerics import mul_rn_f32
from sglang.kernels.ops.diffusion.common.platform import (
is_cuda,
is_hip,
@@ -18,7 +18,7 @@ import torch
import triton # type: ignore
import triton.language as tl # type: ignore
from sglang.kernels.ops.diffusion.common.numerics import mul_rn_f32
from sglang.kernels.numerics import mul_rn_f32
_MAX_BLOCK = 4096
@@ -55,7 +55,7 @@ import torch
import triton # type: ignore
import triton.language as tl # type: ignore
from sglang.kernels.ops.diffusion.common.numerics import (
from sglang.kernels.numerics import (
mul_rn_f32,
round_bf16_to_fp32,
rsqrt_approx_f32,
@@ -29,7 +29,7 @@ import torch
import triton # type: ignore
import triton.language as tl # type: ignore
from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32
from sglang.kernels.numerics import round_bf16_to_fp32
from sglang.srt.utils.custom_op import register_custom_op
@@ -182,6 +182,8 @@ _TRITON_KERNELS = [
("awq_triton", "awq_dequantize_triton"),
("awq_triton", "awq_gemm_triton"),
("mxfp8_amd_gfx95", "mxfp8_e4m3_quantize"),
("mxfp8_swizzled_triton", "swiglu_oai_mxfp8"),
("mxfp8_swizzled_triton", "swiglu_oai_mxfp8_deepgemm"),
]
for _mod, _fn in _TRITON_KERNELS:
register_kernel(
@@ -18,14 +18,14 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32
from sglang.kernels.numerics import round_bf16_to_fp32
_E4M3 = torch.float8_e4m3fn
def _scale_numel(rows: int, k: int) -> int:
n_groups = k // 32
return -(-rows // 128) * 128 * (-(-n_groups // 4) * 4)
return triton.cdiv(rows, 128) * 128 * (triton.cdiv(n_groups, 4) * 4)
@triton.jit
@@ -96,6 +96,10 @@ def _silu_mul_mxfp8_kernel(
n_groups,
n_col_blocks,
stride_row,
dg_rows,
ALPHA: tl.constexpr,
LIMIT: tl.constexpr,
DEEPGEMM: tl.constexpr,
BLOCK_R: tl.constexpr,
G: tl.constexpr,
):
@@ -109,10 +113,20 @@ def _silu_mul_mxfp8_kernel(
base = x_ptr + r[:, None].to(tl.int64) * stride_row
gate = tl.load(base + c[None, :], mask=mask, other=0.0).to(tl.float32)
up = tl.load(base + hidden + c[None, :], mask=mask, other=0.0).to(tl.float32)
act = (gate * tl.sigmoid(gate)).to(tl.bfloat16).to(tl.float32)
prod = (act * up).to(tl.bfloat16).to(tl.float32)
if ALPHA is None:
act = (gate * tl.sigmoid(gate)).to(tl.bfloat16).to(tl.float32)
prod = (act * up).to(tl.bfloat16).to(tl.float32)
else:
# the op order Inductor emits for swiglu_no_interleaved_with_alpha_and_limit
gate = tl.minimum(gate, LIMIT, propagate_nan=tl.PropagateNan.ALL)
up = tl.maximum(up, -LIMIT, propagate_nan=tl.PropagateNan.ALL)
up = tl.minimum(up, LIMIT, propagate_nan=tl.PropagateNan.ALL)
prod = gate * tl.sigmoid(gate * ALPHA) * (up + 1.0)
prod = prod.to(tl.bfloat16).to(tl.float32)
p3 = tl.reshape(prod, [BLOCK_R, G, 32])
amax = tl.max(tl.abs(p3), axis=2)
if DEEPGEMM:
amax = tl.maximum(amax, 1e-10)
sbyte, inv = _mx_e8m0_from_amax(amax)
q = tl.reshape(p3 * inv[:, :, None], [BLOCK_R, G * 32])
tl.store(
@@ -120,12 +134,24 @@ def _silu_mul_mxfp8_kernel(
q.to(tl.float8e4nv),
mask=mask,
)
smask = rmask[:, None] & (g < n_groups)[None, :]
tl.store(
s_ptr + _mx_scale_offsets(r[:, None], g[None, :], n_col_blocks),
sbyte.to(tl.uint8),
mask=smask,
)
valid = rmask[:, None] & (g < n_groups)[None, :]
if DEEPGEMM:
# 4 bytes per int32 along K, MN-major, rows padded to 4
soff = (
(g[None, :] // 4).to(tl.int64) * dg_rows * 4
+ r[:, None] * 4
+ g[None, :] % 4
)
tl.store(s_ptr + soff, sbyte.to(tl.uint8), mask=valid)
else:
smask = (r < tl.cdiv(rows, 128) * 128)[:, None] & (g < n_col_blocks * 4)[
None, :
]
tl.store(
s_ptr + _mx_scale_offsets(r[:, None], g[None, :], n_col_blocks),
tl.where(valid, sbyte, 0).to(tl.uint8),
mask=smask,
)
@triton.jit
@@ -194,10 +220,11 @@ def can_use_mxfp8_swizzled(x: torch.Tensor) -> bool:
def _alloc(
rows: int, k: int, device: torch.device
rows: int, k: int, device: torch.device, zero_scales: bool = True
) -> tuple[torch.Tensor, torch.Tensor]:
q = torch.empty(rows, k, dtype=_E4M3, device=device)
s = torch.zeros(_scale_numel(rows, k), dtype=torch.uint8, device=device)
alloc = torch.zeros if zero_scales else torch.empty
s = alloc(_scale_numel(rows, k), dtype=torch.uint8, device=device)
return q, s
@@ -210,7 +237,7 @@ def mxfp8_quantize_swizzled(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor
if rows == 0:
return q, s
n_groups = k // 32
n_col_blocks = -(-n_groups // 4)
n_col_blocks = triton.cdiv(n_groups, 4)
block_r, g = 32, 8
grid = (triton.cdiv(rows, block_r), triton.cdiv(n_groups, g))
with torch.get_device_module().device(x.device):
@@ -234,31 +261,52 @@ def can_use_silu_mul_mxfp8(hidden: torch.Tensor) -> bool:
return can_use_mxfp8_swizzled(hidden) and (hidden.shape[-1] // 2) % 32 == 0
def silu_mul_mxfp8(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""``hidden [rows, 2n]`` bf16 (gate | up) -> quantized ``silu(gate) * up``."""
def _swiglu_mxfp8(
hidden: torch.Tensor,
alpha: float | None,
limit: float | None,
deepgemm: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
if not can_use_silu_mul_mxfp8(hidden):
raise ValueError(
"expected a row-major bf16 CUDA [rows, 2 * n] tensor, n % 32 == 0"
)
rows, twice = hidden.shape
n = twice // 2
q, s = _alloc(rows, n, hidden.device)
if deepgemm:
if n % 128:
raise ValueError(f"DeepGEMM MXFP8 scales need n % 128 == 0, got {n=}")
q = torch.empty(rows, n, dtype=_E4M3, device=hidden.device)
dg_rows = triton.cdiv(rows, 4) * 4
base = torch.empty(n // 128, dg_rows, dtype=torch.int32, device=hidden.device)
s = base.T[:rows]
s_bytes = base.view(torch.uint8)
padded_rows = rows
else:
q, s = _alloc(rows, n, hidden.device, zero_scales=False)
s_bytes = s
dg_rows = 0
padded_rows = triton.cdiv(rows, 128) * 128
if rows == 0:
return q, s
n_groups = n // 32
n_col_blocks = -(-n_groups // 4)
block_r, g = 16, 8
grid = (triton.cdiv(rows, block_r), triton.cdiv(n_groups, g))
n_col_blocks = triton.cdiv(n_groups, 4)
block_r, g = (4 if rows <= 1024 else 16), 8
grid = (triton.cdiv(padded_rows, block_r), triton.cdiv(n_groups, g))
with torch.get_device_module().device(hidden.device):
_silu_mul_mxfp8_kernel[grid](
hidden,
q,
s,
s_bytes,
rows,
n,
n_groups,
n_col_blocks,
hidden.stride(0),
dg_rows,
ALPHA=alpha,
LIMIT=limit,
DEEPGEMM=deepgemm,
BLOCK_R=block_r,
G=g,
num_warps=4,
@@ -266,6 +314,29 @@ def silu_mul_mxfp8(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
return q, s
def silu_mul_mxfp8(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""``hidden [rows, 2n]`` bf16 (gate | up) -> quantized ``silu(gate) * up``."""
return _swiglu_mxfp8(hidden, None, None)
def swiglu_oai_mxfp8(
hidden: torch.Tensor, alpha: float, limit: float
) -> tuple[torch.Tensor, torch.Tensor]:
"""``hidden [rows, 2n]`` bf16 (gate | up) -> quantized SwiGLU-OAI,
``clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``."""
return _swiglu_mxfp8(hidden, alpha, limit)
def swiglu_oai_mxfp8_deepgemm(
hidden: torch.Tensor, alpha: float, limit: float
) -> tuple[torch.Tensor, torch.Tensor]:
"""Same values as ``swiglu_oai_mxfp8``; scales in DeepGEMM's layout, the
``sglang_per_token_group_quant_fp8(..., 32, column_major_scales=True,
scale_tma_aligned=True, scale_ue8m0=True)`` output: int32 ``[rows, n / 128]``,
four UE8M0 bytes each, MN-major with rows padded to 4."""
return _swiglu_mxfp8(hidden, alpha, limit, deepgemm=True)
def indexed_scale_shift_mxfp8_(
x: torch.Tensor,
shift: torch.Tensor,
@@ -285,7 +356,7 @@ def indexed_scale_shift_mxfp8_(
if rows == 0:
return (x if keep_bf16 else None), q, s
n_groups = hidden_size // 32
n_col_blocks = -(-n_groups // 4)
n_col_blocks = triton.cdiv(n_groups, 4)
block_n = triton.next_power_of_2(hidden_size)
with torch.get_device_module().device(x.device):
_indexed_scale_shift_mxfp8_kernel[(rows,)](
@@ -315,4 +386,6 @@ __all__ = [
"indexed_scale_shift_mxfp8_",
"mxfp8_quantize_swizzled",
"silu_mul_mxfp8",
"swiglu_oai_mxfp8",
"swiglu_oai_mxfp8_deepgemm",
]
@@ -878,13 +878,18 @@ def _deepgemm_w8a8_mxfp8_linear_with_fallback(
sglang_per_token_group_quant_fp8,
)
assert input_scale is None
output_dtype = input.dtype
pre_quantized = input_scale is not None
output_dtype = torch.bfloat16 if pre_quantized else input.dtype
shape_supported = weight.shape[0] % 64 == 0 and weight.shape[1] % 128 == 0
dtype_supported = output_dtype == torch.bfloat16
if not (shape_supported and dtype_supported):
if pre_quantized:
raise ValueError(
"Pre-quantized DeepGEMM input requires N % 64 == 0 and K % 128 == 0; "
"its packed scales cannot be used by the FlashInfer fallback."
)
if weight_scale_swizzled is None:
raise RuntimeError(
f"DeepGEMM cannot serve this MXFP8 GEMM ({shape_supported=}, "
@@ -902,13 +907,32 @@ def _deepgemm_w8a8_mxfp8_linear_with_fallback(
input_2d = input.view(-1, input.shape[-1])
output_shape = [*input.shape[:-1], weight.shape[0]]
q_input, x_scale = sglang_per_token_group_quant_fp8(
input_2d,
32,
column_major_scales=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0,
scale_tma_aligned=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0,
scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0,
)
if pre_quantized:
m, k = input_2d.shape
if not (
deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
and input.ndim == 2
and input.is_contiguous()
and input.dtype == torch.float8_e4m3fn
and input_scale.dtype == torch.int32
and input_scale.device == input.device
and input_scale.shape == (m, k // 128)
and input_scale.stride() == (1, ceil_align(m, 4))
and k == weight.shape[1]
):
raise ValueError(
"Expected 2D contiguous FP8 input with packed, MN-major, "
"TMA-aligned UE8M0 scales for DeepGEMM MXFP8."
)
q_input, x_scale = input_2d, input_scale
else:
q_input, x_scale = sglang_per_token_group_quant_fp8(
input_2d,
32,
column_major_scales=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0,
scale_tma_aligned=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0,
scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0,
)
# weight_scale format is set per-backend by _process_mxfp8_linear_weight_scale
# (int32 packed TMA-aligned on Blackwell, float32 on Hopper); NOT uint8 — Triton form is routed to the fallback above.
@@ -14,3 +14,40 @@ class Mxfp8SwizzledInput(NamedTuple):
data: torch.Tensor
scales: torch.Tensor
def accepts_mxfp8_swizzled_input(linear) -> bool:
"""Whether a linear can consume the explicit 128x4 producer contract."""
qm = getattr(linear, "quant_method", None)
backend = getattr(qm, "mxfp8_dense_backend", None)
return bool(
backend is not None
and not getattr(qm, "use_marlin", False)
and (
getattr(qm, "use_mxfp8", False)
or (
getattr(qm, "block_fp8_as_mxfp8", False)
and getattr(linear, "block_fp8_mxfp8_ready", False)
)
)
and (backend.is_flashinfer_cutlass() or backend.is_flashinfer_cutedsl())
)
def accepts_mxfp8_deepgemm_input(linear) -> bool:
"""Whether a linear takes a ``(q, scale)`` pair with DeepGEMM's packed UE8M0 scales."""
from sglang.srt.layers import deep_gemm_wrapper
qm = getattr(linear, "quant_method", None)
backend = getattr(qm, "mxfp8_dense_backend", None)
weight = getattr(linear, "weight", None)
return bool(
backend is not None
and backend.is_deep_gemm()
and getattr(qm, "use_mxfp8", False)
and not getattr(qm, "use_marlin", False)
and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
and weight is not None
and weight.shape[0] % 64 == 0
and weight.shape[1] % 128 == 0
)
+37 -4
View File
@@ -23,6 +23,11 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.kernels.ops.quantization.mxfp8_swizzled_triton import (
can_use_silu_mul_mxfp8,
swiglu_oai_mxfp8,
swiglu_oai_mxfp8_deepgemm,
)
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
from sglang.srt.configs.model_config import (
get_minimax_sparse_disable_value_layer_ids,
@@ -64,6 +69,11 @@ from sglang.srt.layers.moe.utils import (
is_shared_experts_fusion_disabled,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.mxfp8_input import (
Mxfp8SwizzledInput,
accepts_mxfp8_deepgemm_input,
accepts_mxfp8_swizzled_input,
)
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.utils import PPMissingLayer
@@ -90,7 +100,7 @@ from sglang.srt.model_loader.weight_utils import (
from sglang.srt.models.deepseek_common.utils import tiny_router_gemm_max_tokens
from sglang.srt.models.minimax_m2 import MiniMaxM2RMSNormTP
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream
from sglang.srt.utils import (
add_prefix,
get_device_sm,
@@ -286,6 +296,7 @@ class MiniMaxM3MLP(nn.Module):
super().__init__()
hidden_size = config.hidden_size
hidden_act = config.hidden_act
self.fuse_swiglu_oai_mxfp8 = hidden_act == "swigluoai"
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
@@ -307,9 +318,11 @@ class MiniMaxM3MLP(nn.Module):
if hidden_act == "silu":
self.act_fn = SiluAndMul()
elif hidden_act == "swigluoai":
self.swiglu_alpha = float(config.swiglu_alpha)
self.swiglu_limit = float(config.swiglu_limit)
if _is_npu:
self.act_fn = lambda x: self._swigluoai_fused(
x, config.swiglu_alpha, config.swiglu_limit
x, self.swiglu_alpha, self.swiglu_limit
)
else:
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import (
@@ -317,20 +330,40 @@ class MiniMaxM3MLP(nn.Module):
)
self.act_fn = lambda x: swiglu_no_interleaved_with_alpha_and_limit(
x, config.swiglu_alpha, config.swiglu_limit
x, self.swiglu_alpha, self.swiglu_limit
)
else:
raise ValueError(
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
)
def _swiglu_oai_mxfp8(self, gate_up: torch.Tensor):
if not (
self.fuse_swiglu_oai_mxfp8
and gate_up.shape[0] > 0
and not get_forward().sp_active
and can_use_silu_mul_mxfp8(gate_up)
):
return None
if accepts_mxfp8_swizzled_input(self.down_proj):
return Mxfp8SwizzledInput(
*swiglu_oai_mxfp8(gate_up, self.swiglu_alpha, self.swiglu_limit)
)
if accepts_mxfp8_deepgemm_input(self.down_proj):
return swiglu_oai_mxfp8_deepgemm(
gate_up, self.swiglu_alpha, self.swiglu_limit
)
return None
def forward(
self,
x,
forward_batch: Optional[ForwardBatch] = None,
):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x = self._swiglu_oai_mxfp8(gate_up)
if x is None:
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x