From 6acaf3374abac87f7410358c2832a5f9fe357cf4 Mon Sep 17 00:00:00 2001 From: Chunan Zeng Date: Fri, 9 Oct 2026 17:51:57 -0700 Subject: [PATCH] [kernel] SwiGLU-OAI MXFP8 producer; move swizzled MXFP8 producers to ops/quantization (#42810) Co-authored-by: Claude Opus 5.5 Co-authored-by: Denji-kk <51020465+Denji-kk@users.noreply.github.com> --- .../kda_kernels/layernorm_modulate_triton.py | 2 +- .../{ops/diffusion/common => }/numerics.py | 2 +- python/sglang/kernels/ops/diffusion/README.md | 6 +- .../sglang/kernels/ops/diffusion/__init__.py | 18 +-- .../activation/sana_conv_post_triton.py | 2 +- .../diffusion/activation/silu_mul_bitexact.py | 2 +- .../kernels/ops/diffusion/common/__init__.py | 1 - .../modulate/indexed_modulation_triton.py | 2 +- .../diffusion/modulate/scale_shift_triton.py | 2 +- .../ops/diffusion/norm/h3_vae_fused_norm.py | 2 +- .../norm/rmsnorm_scale_shift_bitexact.py | 2 +- .../rope/rope_rotate_half_bitexact.py | 2 +- .../kernels/ops/quantization/__init__.py | 2 + .../quantization/mxfp8_swizzled_triton.py | 115 +++++++++++--- .../srt/layers/quantization/fp8_utils.py | 42 +++-- .../srt/layers/quantization/mxfp8_input.py | 37 +++++ python/sglang/srt/models/minimax_m3.py | 41 ++++- .../ops/diffusion/test_mxfp8_swizzled.py | 78 --------- .../ops/quantization/test_mxfp8_swizzled.py | 150 ++++++++++++++++++ 19 files changed, 375 insertions(+), 133 deletions(-) rename python/sglang/kernels/{ops/diffusion/common => }/numerics.py (95%) rename python/sglang/kernels/ops/{diffusion => }/quantization/mxfp8_swizzled_triton.py (70%) delete mode 100644 test/registered/kernels/ops/diffusion/test_mxfp8_swizzled.py create mode 100644 test/registered/kernels/ops/quantization/test_mxfp8_swizzled.py diff --git a/python/sglang/kernels/kda_kernels/layernorm_modulate_triton.py b/python/sglang/kernels/kda_kernels/layernorm_modulate_triton.py index 394cb50145..cc18c60264 100644 --- a/python/sglang/kernels/kda_kernels/layernorm_modulate_triton.py +++ b/python/sglang/kernels/kda_kernels/layernorm_modulate_triton.py @@ -48,7 +48,7 @@ import triton # type: ignore import triton.language as tl # type: ignore from sglang.kernels.jit.utils import get_jit_cuda_arch -from sglang.kernels.ops.diffusion.common.numerics import ( +from sglang.kernels.numerics import ( cuda_rsqrtf, div_rn_f32, round_bf16_to_fp32, diff --git a/python/sglang/kernels/ops/diffusion/common/numerics.py b/python/sglang/kernels/numerics.py similarity index 95% rename from python/sglang/kernels/ops/diffusion/common/numerics.py rename to python/sglang/kernels/numerics.py index 37c18a7743..704988959b 100644 --- a/python/sglang/kernels/ops/diffusion/common/numerics.py +++ b/python/sglang/kernels/numerics.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""Numerical primitives shared by bit-exact diffusion Triton kernels.""" +"""Numerical primitives shared by bit-exact Triton kernels.""" import triton # type: ignore import triton.language as tl # type: ignore diff --git a/python/sglang/kernels/ops/diffusion/README.md b/python/sglang/kernels/ops/diffusion/README.md index 976993c3eb..6c603492df 100644 --- a/python/sglang/kernels/ops/diffusion/README.md +++ b/python/sglang/kernels/ops/diffusion/README.md @@ -37,11 +37,11 @@ norm/ RMSNorm / LayerNorm / GroupNorm and their fused epilogues modulate/ adaLN modulate, gating, timestep conditioning rope/ rotary embeddings and the QK-norm chains fused into them activation/ SiLU / GLU / GELU fusions -quantization/ MXFP8 producers whose scales land in the GEMM's swizzled layout +quantization/ FP8 rowwise producers fused into the GEMM input path attention/ sparse linear attention, gated delta-net routing/ diffusion-model MoE routing and expert selection layout/ pure data movement: USP/Ulysses relayout, varlen pack, causal pad -common/ numerics primitives, platform predicates, non-Triton fallbacks +common/ platform predicates, non-Triton fallbacks (numerics: kernels/numerics.py) sites/ request-scoped mount policy — NOT kernels (see below) ext/ JIT C++/CUDA extensions (Hunyuan3D raster/inpaint) — NOT kernels ../../kda_kernels/ agent-generated implementations and their JIT CUDA sources @@ -192,6 +192,8 @@ tensor copy at each residual site. ### MXFP8 producers (online `mxfp8`, cuBLASLt block-scaled GEMM on SM100) +Defined in `sglang.kernels.ops.quantization.mxfp8_swizzled_triton` and re-exported here. + | Entry point | Backend | Contract | |---|---|---| | `mxfp8_quantize_swizzled` | Triton | bit-exact vs `flashinfer.mxfp8_quantize(x, True)`: e4m3 payload + block-32 E8M0 scales in the `SWIZZLE_32_4_4` layout; weights at load and any bf16 GEMM input | diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index e242f126af..27fa0211ff 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -13,7 +13,7 @@ Layout -- ordinary implementations use one subpackage per **operator domain** ``layout``). Implementations generated by kernel-design agents live in ``sglang.kernels.kda_kernels`` and are still exported through this facade. -``common`` holds shared numerics and platform plumbing, ``sites`` the +``common`` holds platform plumbing, ``sites`` the request-scoped mount policy, and ``ext`` JIT C++/CUDA extensions that are not kernels. Start from ``README.md``: several norms look interchangeable and are not. @@ -486,21 +486,21 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = ( ( "diffusion.mxfp8_quantize_swizzled", KernelBackend.TRITON, - "quantization.mxfp8_swizzled_triton:mxfp8_quantize_swizzled", + "sglang.kernels.ops.quantization.mxfp8_swizzled_triton:mxfp8_quantize_swizzled", _CUDA, "bf16 -> MXFP8 (e4m3, block-32 E8M0 scales in the cuBLASLt swizzled layout).", ), ( "diffusion.silu_mul_mxfp8", KernelBackend.TRITON, - "quantization.mxfp8_swizzled_triton:silu_mul_mxfp8", + "sglang.kernels.ops.quantization.mxfp8_swizzled_triton:silu_mul_mxfp8", _CUDA, "SwiGLU + MXFP8 quant for the online mxfp8 fc2 input.", ), ( "diffusion.indexed_scale_shift_mxfp8_", KernelBackend.TRITON, - "quantization.mxfp8_swizzled_triton:indexed_scale_shift_mxfp8_", + "sglang.kernels.ops.quantization.mxfp8_swizzled_triton:indexed_scale_shift_mxfp8_", _CUDA, "Indexed adaLN modulation + MXFP8 quant for the online mxfp8 qkv/fc1 inputs.", ), @@ -745,11 +745,11 @@ _EXPORTS: dict[str, str] = { "vdn_frame_stats_prep": "attention.vdn_linear_branch_triton", "vdn_gather_linear_state": "attention.vdn_linear_branch_triton", "vdn_linear_epilogue": "attention.vdn_linear_branch_triton", - "can_use_mxfp8_swizzled": "quantization.mxfp8_swizzled_triton", - "can_use_silu_mul_mxfp8": "quantization.mxfp8_swizzled_triton", - "indexed_scale_shift_mxfp8_": "quantization.mxfp8_swizzled_triton", - "mxfp8_quantize_swizzled": "quantization.mxfp8_swizzled_triton", - "silu_mul_mxfp8": "quantization.mxfp8_swizzled_triton", + "can_use_mxfp8_swizzled": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton", + "can_use_silu_mul_mxfp8": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton", + "indexed_scale_shift_mxfp8_": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton", + "mxfp8_quantize_swizzled": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton", + "silu_mul_mxfp8": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton", "can_use_usp_merge_heads": "layout.usp_relayout_jit", "usp_merge_heads": "layout.usp_relayout_jit", "build_inv_indices": "layout.varlen_pack_pad_triton", diff --git a/python/sglang/kernels/ops/diffusion/activation/sana_conv_post_triton.py b/python/sglang/kernels/ops/diffusion/activation/sana_conv_post_triton.py index 43a6738845..0892f7af6d 100644 --- a/python/sglang/kernels/ops/diffusion/activation/sana_conv_post_triton.py +++ b/python/sglang/kernels/ops/diffusion/activation/sana_conv_post_triton.py @@ -7,7 +7,7 @@ import torch import triton # type: ignore import triton.language as tl # type: ignore -from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32 +from sglang.kernels.numerics import round_bf16_to_fp32 @triton.jit diff --git a/python/sglang/kernels/ops/diffusion/activation/silu_mul_bitexact.py b/python/sglang/kernels/ops/diffusion/activation/silu_mul_bitexact.py index c8cb8c25cc..a521232ba7 100644 --- a/python/sglang/kernels/ops/diffusion/activation/silu_mul_bitexact.py +++ b/python/sglang/kernels/ops/diffusion/activation/silu_mul_bitexact.py @@ -20,7 +20,7 @@ import torch import triton # type: ignore import triton.language as tl # type: ignore -from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32 +from sglang.kernels.numerics import round_bf16_to_fp32 from sglang.srt.utils.custom_op import register_custom_op diff --git a/python/sglang/kernels/ops/diffusion/common/__init__.py b/python/sglang/kernels/ops/diffusion/common/__init__.py index 3a29cfaaf0..e7833efca9 100644 --- a/python/sglang/kernels/ops/diffusion/common/__init__.py +++ b/python/sglang/kernels/ops/diffusion/common/__init__.py @@ -1,6 +1,5 @@ """Shared infrastructure for the diffusion kernels -- no kernels of its own. -- ``numerics`` : rounding/opmath primitives the bit-exact kernels are built from - ``platform`` : device predicates and the Triton-vs-fallback selector - ``fallback_*``: pure-torch / NPU / MPS implementations for Triton-less devices """ diff --git a/python/sglang/kernels/ops/diffusion/modulate/indexed_modulation_triton.py b/python/sglang/kernels/ops/diffusion/modulate/indexed_modulation_triton.py index 5a7853d383..5ebab8178b 100644 --- a/python/sglang/kernels/ops/diffusion/modulate/indexed_modulation_triton.py +++ b/python/sglang/kernels/ops/diffusion/modulate/indexed_modulation_triton.py @@ -4,7 +4,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32 +from sglang.kernels.numerics import round_bf16_to_fp32 # Column tiles per row. One program per whole row padded to a power of two # masked off 37.5% of MiniMax-H3's 5376-wide rows; 2048-wide tiles run both diff --git a/python/sglang/kernels/ops/diffusion/modulate/scale_shift_triton.py b/python/sglang/kernels/ops/diffusion/modulate/scale_shift_triton.py index 222fb14262..127566c7b1 100644 --- a/python/sglang/kernels/ops/diffusion/modulate/scale_shift_triton.py +++ b/python/sglang/kernels/ops/diffusion/modulate/scale_shift_triton.py @@ -2,10 +2,10 @@ import torch import triton # type: ignore import triton.language as tl # type: ignore +from sglang.kernels.numerics import mul_rn_f32 from sglang.kernels.ops.diffusion.common.fallback_torch import ( fuse_scale_shift_kernel_native, ) -from sglang.kernels.ops.diffusion.common.numerics import mul_rn_f32 from sglang.kernels.ops.diffusion.common.platform import ( is_cuda, is_hip, diff --git a/python/sglang/kernels/ops/diffusion/norm/h3_vae_fused_norm.py b/python/sglang/kernels/ops/diffusion/norm/h3_vae_fused_norm.py index ff58b910da..46664de81b 100644 --- a/python/sglang/kernels/ops/diffusion/norm/h3_vae_fused_norm.py +++ b/python/sglang/kernels/ops/diffusion/norm/h3_vae_fused_norm.py @@ -18,7 +18,7 @@ import torch import triton # type: ignore import triton.language as tl # type: ignore -from sglang.kernels.ops.diffusion.common.numerics import mul_rn_f32 +from sglang.kernels.numerics import mul_rn_f32 _MAX_BLOCK = 4096 diff --git a/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_bitexact.py b/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_bitexact.py index b6082dda39..81f3ed7ae6 100644 --- a/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_bitexact.py +++ b/python/sglang/kernels/ops/diffusion/norm/rmsnorm_scale_shift_bitexact.py @@ -55,7 +55,7 @@ import torch import triton # type: ignore import triton.language as tl # type: ignore -from sglang.kernels.ops.diffusion.common.numerics import ( +from sglang.kernels.numerics import ( mul_rn_f32, round_bf16_to_fp32, rsqrt_approx_f32, diff --git a/python/sglang/kernels/ops/diffusion/rope/rope_rotate_half_bitexact.py b/python/sglang/kernels/ops/diffusion/rope/rope_rotate_half_bitexact.py index f574452ccf..387d0528c9 100644 --- a/python/sglang/kernels/ops/diffusion/rope/rope_rotate_half_bitexact.py +++ b/python/sglang/kernels/ops/diffusion/rope/rope_rotate_half_bitexact.py @@ -29,7 +29,7 @@ import torch import triton # type: ignore import triton.language as tl # type: ignore -from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32 +from sglang.kernels.numerics import round_bf16_to_fp32 from sglang.srt.utils.custom_op import register_custom_op diff --git a/python/sglang/kernels/ops/quantization/__init__.py b/python/sglang/kernels/ops/quantization/__init__.py index 06ad2a9358..ea7775c7fd 100644 --- a/python/sglang/kernels/ops/quantization/__init__.py +++ b/python/sglang/kernels/ops/quantization/__init__.py @@ -182,6 +182,8 @@ _TRITON_KERNELS = [ ("awq_triton", "awq_dequantize_triton"), ("awq_triton", "awq_gemm_triton"), ("mxfp8_amd_gfx95", "mxfp8_e4m3_quantize"), + ("mxfp8_swizzled_triton", "swiglu_oai_mxfp8"), + ("mxfp8_swizzled_triton", "swiglu_oai_mxfp8_deepgemm"), ] for _mod, _fn in _TRITON_KERNELS: register_kernel( diff --git a/python/sglang/kernels/ops/diffusion/quantization/mxfp8_swizzled_triton.py b/python/sglang/kernels/ops/quantization/mxfp8_swizzled_triton.py similarity index 70% rename from python/sglang/kernels/ops/diffusion/quantization/mxfp8_swizzled_triton.py rename to python/sglang/kernels/ops/quantization/mxfp8_swizzled_triton.py index 13ad76df6d..0dc789233d 100644 --- a/python/sglang/kernels/ops/diffusion/quantization/mxfp8_swizzled_triton.py +++ b/python/sglang/kernels/ops/quantization/mxfp8_swizzled_triton.py @@ -18,14 +18,14 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32 +from sglang.kernels.numerics import round_bf16_to_fp32 _E4M3 = torch.float8_e4m3fn def _scale_numel(rows: int, k: int) -> int: n_groups = k // 32 - return -(-rows // 128) * 128 * (-(-n_groups // 4) * 4) + return triton.cdiv(rows, 128) * 128 * (triton.cdiv(n_groups, 4) * 4) @triton.jit @@ -96,6 +96,10 @@ def _silu_mul_mxfp8_kernel( n_groups, n_col_blocks, stride_row, + dg_rows, + ALPHA: tl.constexpr, + LIMIT: tl.constexpr, + DEEPGEMM: tl.constexpr, BLOCK_R: tl.constexpr, G: tl.constexpr, ): @@ -109,10 +113,20 @@ def _silu_mul_mxfp8_kernel( base = x_ptr + r[:, None].to(tl.int64) * stride_row gate = tl.load(base + c[None, :], mask=mask, other=0.0).to(tl.float32) up = tl.load(base + hidden + c[None, :], mask=mask, other=0.0).to(tl.float32) - act = (gate * tl.sigmoid(gate)).to(tl.bfloat16).to(tl.float32) - prod = (act * up).to(tl.bfloat16).to(tl.float32) + if ALPHA is None: + act = (gate * tl.sigmoid(gate)).to(tl.bfloat16).to(tl.float32) + prod = (act * up).to(tl.bfloat16).to(tl.float32) + else: + # the op order Inductor emits for swiglu_no_interleaved_with_alpha_and_limit + gate = tl.minimum(gate, LIMIT, propagate_nan=tl.PropagateNan.ALL) + up = tl.maximum(up, -LIMIT, propagate_nan=tl.PropagateNan.ALL) + up = tl.minimum(up, LIMIT, propagate_nan=tl.PropagateNan.ALL) + prod = gate * tl.sigmoid(gate * ALPHA) * (up + 1.0) + prod = prod.to(tl.bfloat16).to(tl.float32) p3 = tl.reshape(prod, [BLOCK_R, G, 32]) amax = tl.max(tl.abs(p3), axis=2) + if DEEPGEMM: + amax = tl.maximum(amax, 1e-10) sbyte, inv = _mx_e8m0_from_amax(amax) q = tl.reshape(p3 * inv[:, :, None], [BLOCK_R, G * 32]) tl.store( @@ -120,12 +134,24 @@ def _silu_mul_mxfp8_kernel( q.to(tl.float8e4nv), mask=mask, ) - smask = rmask[:, None] & (g < n_groups)[None, :] - tl.store( - s_ptr + _mx_scale_offsets(r[:, None], g[None, :], n_col_blocks), - sbyte.to(tl.uint8), - mask=smask, - ) + valid = rmask[:, None] & (g < n_groups)[None, :] + if DEEPGEMM: + # 4 bytes per int32 along K, MN-major, rows padded to 4 + soff = ( + (g[None, :] // 4).to(tl.int64) * dg_rows * 4 + + r[:, None] * 4 + + g[None, :] % 4 + ) + tl.store(s_ptr + soff, sbyte.to(tl.uint8), mask=valid) + else: + smask = (r < tl.cdiv(rows, 128) * 128)[:, None] & (g < n_col_blocks * 4)[ + None, : + ] + tl.store( + s_ptr + _mx_scale_offsets(r[:, None], g[None, :], n_col_blocks), + tl.where(valid, sbyte, 0).to(tl.uint8), + mask=smask, + ) @triton.jit @@ -194,10 +220,11 @@ def can_use_mxfp8_swizzled(x: torch.Tensor) -> bool: def _alloc( - rows: int, k: int, device: torch.device + rows: int, k: int, device: torch.device, zero_scales: bool = True ) -> tuple[torch.Tensor, torch.Tensor]: q = torch.empty(rows, k, dtype=_E4M3, device=device) - s = torch.zeros(_scale_numel(rows, k), dtype=torch.uint8, device=device) + alloc = torch.zeros if zero_scales else torch.empty + s = alloc(_scale_numel(rows, k), dtype=torch.uint8, device=device) return q, s @@ -210,7 +237,7 @@ def mxfp8_quantize_swizzled(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor if rows == 0: return q, s n_groups = k // 32 - n_col_blocks = -(-n_groups // 4) + n_col_blocks = triton.cdiv(n_groups, 4) block_r, g = 32, 8 grid = (triton.cdiv(rows, block_r), triton.cdiv(n_groups, g)) with torch.get_device_module().device(x.device): @@ -234,31 +261,52 @@ def can_use_silu_mul_mxfp8(hidden: torch.Tensor) -> bool: return can_use_mxfp8_swizzled(hidden) and (hidden.shape[-1] // 2) % 32 == 0 -def silu_mul_mxfp8(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - """``hidden [rows, 2n]`` bf16 (gate | up) -> quantized ``silu(gate) * up``.""" +def _swiglu_mxfp8( + hidden: torch.Tensor, + alpha: float | None, + limit: float | None, + deepgemm: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: if not can_use_silu_mul_mxfp8(hidden): raise ValueError( "expected a row-major bf16 CUDA [rows, 2 * n] tensor, n % 32 == 0" ) rows, twice = hidden.shape n = twice // 2 - q, s = _alloc(rows, n, hidden.device) + if deepgemm: + if n % 128: + raise ValueError(f"DeepGEMM MXFP8 scales need n % 128 == 0, got {n=}") + q = torch.empty(rows, n, dtype=_E4M3, device=hidden.device) + dg_rows = triton.cdiv(rows, 4) * 4 + base = torch.empty(n // 128, dg_rows, dtype=torch.int32, device=hidden.device) + s = base.T[:rows] + s_bytes = base.view(torch.uint8) + padded_rows = rows + else: + q, s = _alloc(rows, n, hidden.device, zero_scales=False) + s_bytes = s + dg_rows = 0 + padded_rows = triton.cdiv(rows, 128) * 128 if rows == 0: return q, s n_groups = n // 32 - n_col_blocks = -(-n_groups // 4) - block_r, g = 16, 8 - grid = (triton.cdiv(rows, block_r), triton.cdiv(n_groups, g)) + n_col_blocks = triton.cdiv(n_groups, 4) + block_r, g = (4 if rows <= 1024 else 16), 8 + grid = (triton.cdiv(padded_rows, block_r), triton.cdiv(n_groups, g)) with torch.get_device_module().device(hidden.device): _silu_mul_mxfp8_kernel[grid]( hidden, q, - s, + s_bytes, rows, n, n_groups, n_col_blocks, hidden.stride(0), + dg_rows, + ALPHA=alpha, + LIMIT=limit, + DEEPGEMM=deepgemm, BLOCK_R=block_r, G=g, num_warps=4, @@ -266,6 +314,29 @@ def silu_mul_mxfp8(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: return q, s +def silu_mul_mxfp8(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """``hidden [rows, 2n]`` bf16 (gate | up) -> quantized ``silu(gate) * up``.""" + return _swiglu_mxfp8(hidden, None, None) + + +def swiglu_oai_mxfp8( + hidden: torch.Tensor, alpha: float, limit: float +) -> tuple[torch.Tensor, torch.Tensor]: + """``hidden [rows, 2n]`` bf16 (gate | up) -> quantized SwiGLU-OAI, + ``clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``.""" + return _swiglu_mxfp8(hidden, alpha, limit) + + +def swiglu_oai_mxfp8_deepgemm( + hidden: torch.Tensor, alpha: float, limit: float +) -> tuple[torch.Tensor, torch.Tensor]: + """Same values as ``swiglu_oai_mxfp8``; scales in DeepGEMM's layout, the + ``sglang_per_token_group_quant_fp8(..., 32, column_major_scales=True, + scale_tma_aligned=True, scale_ue8m0=True)`` output: int32 ``[rows, n / 128]``, + four UE8M0 bytes each, MN-major with rows padded to 4.""" + return _swiglu_mxfp8(hidden, alpha, limit, deepgemm=True) + + def indexed_scale_shift_mxfp8_( x: torch.Tensor, shift: torch.Tensor, @@ -285,7 +356,7 @@ def indexed_scale_shift_mxfp8_( if rows == 0: return (x if keep_bf16 else None), q, s n_groups = hidden_size // 32 - n_col_blocks = -(-n_groups // 4) + n_col_blocks = triton.cdiv(n_groups, 4) block_n = triton.next_power_of_2(hidden_size) with torch.get_device_module().device(x.device): _indexed_scale_shift_mxfp8_kernel[(rows,)]( @@ -315,4 +386,6 @@ __all__ = [ "indexed_scale_shift_mxfp8_", "mxfp8_quantize_swizzled", "silu_mul_mxfp8", + "swiglu_oai_mxfp8", + "swiglu_oai_mxfp8_deepgemm", ] diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 2274f3c0c9..227eaea3bb 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -878,13 +878,18 @@ def _deepgemm_w8a8_mxfp8_linear_with_fallback( sglang_per_token_group_quant_fp8, ) - assert input_scale is None - output_dtype = input.dtype + pre_quantized = input_scale is not None + output_dtype = torch.bfloat16 if pre_quantized else input.dtype shape_supported = weight.shape[0] % 64 == 0 and weight.shape[1] % 128 == 0 dtype_supported = output_dtype == torch.bfloat16 if not (shape_supported and dtype_supported): + if pre_quantized: + raise ValueError( + "Pre-quantized DeepGEMM input requires N % 64 == 0 and K % 128 == 0; " + "its packed scales cannot be used by the FlashInfer fallback." + ) if weight_scale_swizzled is None: raise RuntimeError( f"DeepGEMM cannot serve this MXFP8 GEMM ({shape_supported=}, " @@ -902,13 +907,32 @@ def _deepgemm_w8a8_mxfp8_linear_with_fallback( input_2d = input.view(-1, input.shape[-1]) output_shape = [*input.shape[:-1], weight.shape[0]] - q_input, x_scale = sglang_per_token_group_quant_fp8( - input_2d, - 32, - column_major_scales=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, - scale_tma_aligned=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, - scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, - ) + if pre_quantized: + m, k = input_2d.shape + if not ( + deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 + and input.ndim == 2 + and input.is_contiguous() + and input.dtype == torch.float8_e4m3fn + and input_scale.dtype == torch.int32 + and input_scale.device == input.device + and input_scale.shape == (m, k // 128) + and input_scale.stride() == (1, ceil_align(m, 4)) + and k == weight.shape[1] + ): + raise ValueError( + "Expected 2D contiguous FP8 input with packed, MN-major, " + "TMA-aligned UE8M0 scales for DeepGEMM MXFP8." + ) + q_input, x_scale = input_2d, input_scale + else: + q_input, x_scale = sglang_per_token_group_quant_fp8( + input_2d, + 32, + column_major_scales=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + scale_tma_aligned=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + ) # weight_scale format is set per-backend by _process_mxfp8_linear_weight_scale # (int32 packed TMA-aligned on Blackwell, float32 on Hopper); NOT uint8 — Triton form is routed to the fallback above. diff --git a/python/sglang/srt/layers/quantization/mxfp8_input.py b/python/sglang/srt/layers/quantization/mxfp8_input.py index 326bfe2dc6..4f9838b7d0 100644 --- a/python/sglang/srt/layers/quantization/mxfp8_input.py +++ b/python/sglang/srt/layers/quantization/mxfp8_input.py @@ -14,3 +14,40 @@ class Mxfp8SwizzledInput(NamedTuple): data: torch.Tensor scales: torch.Tensor + + +def accepts_mxfp8_swizzled_input(linear) -> bool: + """Whether a linear can consume the explicit 128x4 producer contract.""" + qm = getattr(linear, "quant_method", None) + backend = getattr(qm, "mxfp8_dense_backend", None) + return bool( + backend is not None + and not getattr(qm, "use_marlin", False) + and ( + getattr(qm, "use_mxfp8", False) + or ( + getattr(qm, "block_fp8_as_mxfp8", False) + and getattr(linear, "block_fp8_mxfp8_ready", False) + ) + ) + and (backend.is_flashinfer_cutlass() or backend.is_flashinfer_cutedsl()) + ) + + +def accepts_mxfp8_deepgemm_input(linear) -> bool: + """Whether a linear takes a ``(q, scale)`` pair with DeepGEMM's packed UE8M0 scales.""" + from sglang.srt.layers import deep_gemm_wrapper + + qm = getattr(linear, "quant_method", None) + backend = getattr(qm, "mxfp8_dense_backend", None) + weight = getattr(linear, "weight", None) + return bool( + backend is not None + and backend.is_deep_gemm() + and getattr(qm, "use_mxfp8", False) + and not getattr(qm, "use_marlin", False) + and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 + and weight is not None + and weight.shape[0] % 64 == 0 + and weight.shape[1] % 128 == 0 + ) diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 36494c9782..860bfcbb7b 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -23,6 +23,11 @@ import torch from torch import nn from transformers import PretrainedConfig +from sglang.kernels.ops.quantization.mxfp8_swizzled_triton import ( + can_use_silu_mul_mxfp8, + swiglu_oai_mxfp8, + swiglu_oai_mxfp8_deepgemm, +) from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages from sglang.srt.configs.model_config import ( get_minimax_sparse_disable_value_layer_ids, @@ -64,6 +69,11 @@ from sglang.srt.layers.moe.utils import ( is_shared_experts_fusion_disabled, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.quantization.mxfp8_input import ( + Mxfp8SwizzledInput, + accepts_mxfp8_deepgemm_input, + accepts_mxfp8_swizzled_input, +) from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope from sglang.srt.layers.utils import PPMissingLayer @@ -90,7 +100,7 @@ from sglang.srt.model_loader.weight_utils import ( from sglang.srt.models.deepseek_common.utils import tiny_router_gemm_max_tokens from sglang.srt.models.minimax_m2 import MiniMaxM2RMSNormTP from sglang.srt.models.utils import WeightsMapper -from sglang.srt.runtime_context import get_exec, get_parallel, get_stream +from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream from sglang.srt.utils import ( add_prefix, get_device_sm, @@ -286,6 +296,7 @@ class MiniMaxM3MLP(nn.Module): super().__init__() hidden_size = config.hidden_size hidden_act = config.hidden_act + self.fuse_swiglu_oai_mxfp8 = hidden_act == "swigluoai" self.gate_up_proj = MergedColumnParallelLinear( hidden_size, @@ -307,9 +318,11 @@ class MiniMaxM3MLP(nn.Module): if hidden_act == "silu": self.act_fn = SiluAndMul() elif hidden_act == "swigluoai": + self.swiglu_alpha = float(config.swiglu_alpha) + self.swiglu_limit = float(config.swiglu_limit) if _is_npu: self.act_fn = lambda x: self._swigluoai_fused( - x, config.swiglu_alpha, config.swiglu_limit + x, self.swiglu_alpha, self.swiglu_limit ) else: from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import ( @@ -317,20 +330,40 @@ class MiniMaxM3MLP(nn.Module): ) self.act_fn = lambda x: swiglu_no_interleaved_with_alpha_and_limit( - x, config.swiglu_alpha, config.swiglu_limit + x, self.swiglu_alpha, self.swiglu_limit ) else: raise ValueError( f"Unsupported activation: {hidden_act}. Only silu is supported for now." ) + def _swiglu_oai_mxfp8(self, gate_up: torch.Tensor): + if not ( + self.fuse_swiglu_oai_mxfp8 + and gate_up.shape[0] > 0 + and not get_forward().sp_active + and can_use_silu_mul_mxfp8(gate_up) + ): + return None + if accepts_mxfp8_swizzled_input(self.down_proj): + return Mxfp8SwizzledInput( + *swiglu_oai_mxfp8(gate_up, self.swiglu_alpha, self.swiglu_limit) + ) + if accepts_mxfp8_deepgemm_input(self.down_proj): + return swiglu_oai_mxfp8_deepgemm( + gate_up, self.swiglu_alpha, self.swiglu_limit + ) + return None + def forward( self, x, forward_batch: Optional[ForwardBatch] = None, ): gate_up, _ = self.gate_up_proj(x) - x = self.act_fn(gate_up) + x = self._swiglu_oai_mxfp8(gate_up) + if x is None: + x = self.act_fn(gate_up) x, _ = self.down_proj(x) return x diff --git a/test/registered/kernels/ops/diffusion/test_mxfp8_swizzled.py b/test/registered/kernels/ops/diffusion/test_mxfp8_swizzled.py deleted file mode 100644 index ac6debc217..0000000000 --- a/test/registered/kernels/ops/diffusion/test_mxfp8_swizzled.py +++ /dev/null @@ -1,78 +0,0 @@ -"""MXFP8 producers against ``flashinfer.mxfp8_quantize`` of the bf16 tensor the -unfused kernel stores: payload and the swizzled E8M0 scale buffer, padding -included, byte for byte.""" - -import sys - -import pytest -import torch - -from sglang.kernels.ops.diffusion import ( - can_use_mxfp8_swizzled, - can_use_silu_mul_mxfp8, - indexed_scale_shift_bf16_, - indexed_scale_shift_mxfp8_, - mxfp8_quantize_swizzled, - silu_mul_mxfp8, -) -from sglang.test.ci.ci_register import register_cuda_ci - -register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="4-gpu-b200") - -ROWS, HIDDEN = 333, 5376 # rows and scale columns both need padding - - -def _assert_matches_flashinfer( - got: tuple[torch.Tensor, torch.Tensor], bf16: torch.Tensor -) -> None: - import flashinfer - - q, s = flashinfer.mxfp8_quantize(bf16.contiguous(), True) - assert torch.equal(got[0].view(torch.uint8), q.view(torch.uint8)) - assert torch.equal(got[1].view(torch.uint8), s.view(torch.uint8)) - - -def test_quantize_swizzled_is_byte_exact() -> None: - g = torch.Generator(device="cuda").manual_seed(1) - x = (torch.randn((ROWS, HIDDEN), device="cuda", generator=g) * 3).to(torch.bfloat16) - x[0, :32] = 0 # an all-zero block takes the minimum exponent - assert can_use_mxfp8_swizzled(x) - _assert_matches_flashinfer(mxfp8_quantize_swizzled(x), x) - - -def test_silu_mul_mxfp8_is_byte_exact() -> None: - g = torch.Generator(device="cuda").manual_seed(2) - x = (torch.randn((ROWS, 2 * HIDDEN), device="cuda", generator=g) * 2).to( - torch.bfloat16 - ) - assert can_use_silu_mul_mxfp8(x) - ref = torch.nn.functional.silu(x[:, :HIDDEN]) * x[:, HIDDEN:] - _assert_matches_flashinfer(silu_mul_mxfp8(x), ref) - - -@pytest.mark.parametrize("keep_bf16", [True, False]) -def test_indexed_scale_shift_mxfp8_is_byte_exact(keep_bf16: bool) -> None: - g = torch.Generator(device="cuda").manual_seed(3) - x = torch.randn((ROWS, HIDDEN), device="cuda", generator=g).to(torch.bfloat16) - shift = torch.randn((3, HIDDEN), device="cuda", generator=g).to(torch.bfloat16) - scale = torch.randn((3, HIDDEN), device="cuda", generator=g).to(torch.bfloat16) - indices = torch.randint(0, 3, (ROWS,), device="cuda", generator=g) - ref = indexed_scale_shift_bf16_(x.clone(), shift, scale, indices) - kept, q, s = indexed_scale_shift_mxfp8_( - x, shift, scale, indices, keep_bf16=keep_bf16 - ) - _assert_matches_flashinfer((q, s), ref) - assert (kept is x and torch.equal(x, ref)) if keep_bf16 else kept is None - - -def test_predicates_reject_unsupported_input() -> None: - assert not can_use_mxfp8_swizzled( - torch.randn(4, 64, device="cuda", dtype=torch.float16) - ) - assert not can_use_silu_mul_mxfp8( - torch.randn(4, 96, device="cuda", dtype=torch.bfloat16) - ) - - -if __name__ == "__main__": - sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/quantization/test_mxfp8_swizzled.py b/test/registered/kernels/ops/quantization/test_mxfp8_swizzled.py new file mode 100644 index 0000000000..8fb94747d5 --- /dev/null +++ b/test/registered/kernels/ops/quantization/test_mxfp8_swizzled.py @@ -0,0 +1,150 @@ +"""MXFP8 producers against ``flashinfer.mxfp8_quantize`` of the bf16 tensor the +unfused kernel stores: payload and the swizzled E8M0 scale buffer, padding +included, byte for byte.""" + +import sys + +import pytest +import torch + +from sglang.kernels.ops.diffusion import indexed_scale_shift_bf16_ +from sglang.kernels.ops.quantization import mxfp8_swizzled_triton +from sglang.kernels.ops.quantization.fp8_kernel import sglang_per_token_group_quant_fp8 +from sglang.kernels.ops.quantization.mxfp8_swizzled_triton import ( + can_use_mxfp8_swizzled, + can_use_silu_mul_mxfp8, + indexed_scale_shift_mxfp8_, + mxfp8_quantize_swizzled, + silu_mul_mxfp8, + swiglu_oai_mxfp8, + swiglu_oai_mxfp8_deepgemm, +) +from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import ( + swiglu_no_interleaved_with_alpha_and_limit, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +ROWS, HIDDEN = 333, 5376 # rows and scale columns both need padding + + +def _assert_matches_flashinfer( + got: tuple[torch.Tensor, torch.Tensor], bf16: torch.Tensor +) -> None: + import flashinfer + + q, s = flashinfer.mxfp8_quantize(bf16.contiguous(), True) + assert torch.equal(got[0].view(torch.uint8), q.view(torch.uint8)) + assert torch.equal(got[1].view(torch.uint8), s.view(torch.uint8)) + + +def test_quantize_swizzled_is_byte_exact() -> None: + g = torch.Generator(device="cuda").manual_seed(1) + x = (torch.randn((ROWS, HIDDEN), device="cuda", generator=g) * 3).to(torch.bfloat16) + x[0, :32] = 0 # an all-zero block takes the minimum exponent + assert can_use_mxfp8_swizzled(x) + _assert_matches_flashinfer(mxfp8_quantize_swizzled(x), x) + + +def test_silu_mul_mxfp8_is_byte_exact(monkeypatch: pytest.MonkeyPatch) -> None: + g = torch.Generator(device="cuda").manual_seed(2) + x = (torch.randn((ROWS, 2 * HIDDEN), device="cuda", generator=g) * 2).to( + torch.bfloat16 + ) + assert can_use_silu_mul_mxfp8(x) + ref = torch.nn.functional.silu(x[:, :HIDDEN]) * x[:, HIDDEN:] + _poison_scale_alloc(monkeypatch) + _assert_matches_flashinfer(silu_mul_mxfp8(x), ref) + + +def _poison_scale_alloc(monkeypatch: pytest.MonkeyPatch) -> None: + alloc = mxfp8_swizzled_triton._alloc + + def poisoned(*args, **kwargs): + q, s = alloc(*args, **kwargs) + return q, s.fill_(0xFF) + + monkeypatch.setattr(mxfp8_swizzled_triton, "_alloc", poisoned) + + +@pytest.mark.parametrize("rows", [1, 7, 128, 333]) +@pytest.mark.parametrize("n", [768, 1056]) +@pytest.mark.parametrize("scale", [1e-3, 1.0, 30.0]) +def test_swiglu_oai_mxfp8_matches_eager_chain( + rows: int, n: int, scale: float, monkeypatch: pytest.MonkeyPatch +) -> None: + g = torch.Generator(device="cuda").manual_seed(rows * 31 + n) + x = (torch.randn((rows, 2 * n + 64), device="cuda", generator=g) * scale).to( + torch.bfloat16 + )[:, : 2 * n] + ref = swiglu_no_interleaved_with_alpha_and_limit(x, 1.702, 7.0) + _poison_scale_alloc(monkeypatch) + _assert_matches_flashinfer(swiglu_oai_mxfp8(x, 1.702, 7.0), ref) + + +def test_swiglu_oai_mxfp8_cuda_graph_replay() -> None: + rows, n = 40, 768 + x = torch.randn((rows, 2 * n), device="cuda").to(torch.bfloat16) + swiglu_oai_mxfp8(x, 1.702, 7.0) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + got = swiglu_oai_mxfp8(x, 1.702, 7.0) + for scale in [0.0, 1e-4, 1.0, 100.0]: + x.copy_(torch.randn_like(x, dtype=torch.float32) * scale) + snapshot = x.clone() + graph.replay() + assert torch.equal(x, snapshot) + ref = swiglu_no_interleaved_with_alpha_and_limit(x, 1.702, 7.0) + _assert_matches_flashinfer(got, ref) + + +@pytest.mark.parametrize("rows", [1, 7, 128, 333, 2050]) +@pytest.mark.parametrize("n", [768, 1536]) +@pytest.mark.parametrize("scale", [0.0, 1e-3, 1.0, 30.0]) +def test_swiglu_oai_mxfp8_deepgemm_matches_eager_chain( + rows: int, n: int, scale: float +) -> None: + g = torch.Generator(device="cuda").manual_seed(rows * 31 + n) + x = (torch.randn((rows, 2 * n + 64), device="cuda", generator=g) * scale).to( + torch.bfloat16 + )[:, : 2 * n] + ref_q, ref_s = sglang_per_token_group_quant_fp8( + swiglu_no_interleaved_with_alpha_and_limit(x, 1.702, 7.0).contiguous(), + 32, + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + ) + q, s = swiglu_oai_mxfp8_deepgemm(x, 1.702, 7.0) + assert s.shape == ref_s.shape and s.stride() == ref_s.stride() + assert torch.equal(q.view(torch.uint8), ref_q.view(torch.uint8)) + assert torch.equal(s, ref_s) + + +@pytest.mark.parametrize("keep_bf16", [True, False]) +def test_indexed_scale_shift_mxfp8_is_byte_exact(keep_bf16: bool) -> None: + g = torch.Generator(device="cuda").manual_seed(3) + x = torch.randn((ROWS, HIDDEN), device="cuda", generator=g).to(torch.bfloat16) + shift = torch.randn((3, HIDDEN), device="cuda", generator=g).to(torch.bfloat16) + scale = torch.randn((3, HIDDEN), device="cuda", generator=g).to(torch.bfloat16) + indices = torch.randint(0, 3, (ROWS,), device="cuda", generator=g) + ref = indexed_scale_shift_bf16_(x.clone(), shift, scale, indices) + kept, q, s = indexed_scale_shift_mxfp8_( + x, shift, scale, indices, keep_bf16=keep_bf16 + ) + _assert_matches_flashinfer((q, s), ref) + assert (kept is x and torch.equal(x, ref)) if keep_bf16 else kept is None + + +def test_predicates_reject_unsupported_input() -> None: + assert not can_use_mxfp8_swizzled( + torch.randn(4, 64, device="cuda", dtype=torch.float16) + ) + assert not can_use_silu_mul_mxfp8( + torch.randn(4, 96, device="cuda", dtype=torch.bfloat16) + ) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))