[kernel] SwiGLU-OAI MXFP8 producer; move swizzled MXFP8 producers to ops/quantization (#42810)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Denji-kk <51020465+Denji-kk@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Denji-kk
parent
961bcf481d
commit
6acaf3374a
@@ -48,7 +48,7 @@ import triton # type: ignore
|
||||
import triton.language as tl # type: ignore
|
||||
|
||||
from sglang.kernels.jit.utils import get_jit_cuda_arch
|
||||
from sglang.kernels.ops.diffusion.common.numerics import (
|
||||
from sglang.kernels.numerics import (
|
||||
cuda_rsqrtf,
|
||||
div_rn_f32,
|
||||
round_bf16_to_fp32,
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Numerical primitives shared by bit-exact diffusion Triton kernels."""
|
||||
"""Numerical primitives shared by bit-exact Triton kernels."""
|
||||
|
||||
import triton # type: ignore
|
||||
import triton.language as tl # type: ignore
|
||||
@@ -37,11 +37,11 @@ norm/ RMSNorm / LayerNorm / GroupNorm and their fused epilogues
|
||||
modulate/ adaLN modulate, gating, timestep conditioning
|
||||
rope/ rotary embeddings and the QK-norm chains fused into them
|
||||
activation/ SiLU / GLU / GELU fusions
|
||||
quantization/ MXFP8 producers whose scales land in the GEMM's swizzled layout
|
||||
quantization/ FP8 rowwise producers fused into the GEMM input path
|
||||
attention/ sparse linear attention, gated delta-net
|
||||
routing/ diffusion-model MoE routing and expert selection
|
||||
layout/ pure data movement: USP/Ulysses relayout, varlen pack, causal pad
|
||||
common/ numerics primitives, platform predicates, non-Triton fallbacks
|
||||
common/ platform predicates, non-Triton fallbacks (numerics: kernels/numerics.py)
|
||||
sites/ request-scoped mount policy — NOT kernels (see below)
|
||||
ext/ JIT C++/CUDA extensions (Hunyuan3D raster/inpaint) — NOT kernels
|
||||
../../kda_kernels/ agent-generated implementations and their JIT CUDA sources
|
||||
@@ -192,6 +192,8 @@ tensor copy at each residual site.
|
||||
|
||||
### MXFP8 producers (online `mxfp8`, cuBLASLt block-scaled GEMM on SM100)
|
||||
|
||||
Defined in `sglang.kernels.ops.quantization.mxfp8_swizzled_triton` and re-exported here.
|
||||
|
||||
| Entry point | Backend | Contract |
|
||||
|---|---|---|
|
||||
| `mxfp8_quantize_swizzled` | Triton | bit-exact vs `flashinfer.mxfp8_quantize(x, True)`: e4m3 payload + block-32 E8M0 scales in the `SWIZZLE_32_4_4` layout; weights at load and any bf16 GEMM input |
|
||||
|
||||
@@ -13,7 +13,7 @@ Layout -- ordinary implementations use one subpackage per **operator domain**
|
||||
``layout``).
|
||||
Implementations generated by kernel-design agents live in
|
||||
``sglang.kernels.kda_kernels`` and are still exported through this facade.
|
||||
``common`` holds shared numerics and platform plumbing, ``sites`` the
|
||||
``common`` holds platform plumbing, ``sites`` the
|
||||
request-scoped mount policy, and ``ext`` JIT C++/CUDA extensions that are not
|
||||
kernels. Start from ``README.md``: several norms look interchangeable and are
|
||||
not.
|
||||
@@ -486,21 +486,21 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
|
||||
(
|
||||
"diffusion.mxfp8_quantize_swizzled",
|
||||
KernelBackend.TRITON,
|
||||
"quantization.mxfp8_swizzled_triton:mxfp8_quantize_swizzled",
|
||||
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:mxfp8_quantize_swizzled",
|
||||
_CUDA,
|
||||
"bf16 -> MXFP8 (e4m3, block-32 E8M0 scales in the cuBLASLt swizzled layout).",
|
||||
),
|
||||
(
|
||||
"diffusion.silu_mul_mxfp8",
|
||||
KernelBackend.TRITON,
|
||||
"quantization.mxfp8_swizzled_triton:silu_mul_mxfp8",
|
||||
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:silu_mul_mxfp8",
|
||||
_CUDA,
|
||||
"SwiGLU + MXFP8 quant for the online mxfp8 fc2 input.",
|
||||
),
|
||||
(
|
||||
"diffusion.indexed_scale_shift_mxfp8_",
|
||||
KernelBackend.TRITON,
|
||||
"quantization.mxfp8_swizzled_triton:indexed_scale_shift_mxfp8_",
|
||||
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:indexed_scale_shift_mxfp8_",
|
||||
_CUDA,
|
||||
"Indexed adaLN modulation + MXFP8 quant for the online mxfp8 qkv/fc1 inputs.",
|
||||
),
|
||||
@@ -745,11 +745,11 @@ _EXPORTS: dict[str, str] = {
|
||||
"vdn_frame_stats_prep": "attention.vdn_linear_branch_triton",
|
||||
"vdn_gather_linear_state": "attention.vdn_linear_branch_triton",
|
||||
"vdn_linear_epilogue": "attention.vdn_linear_branch_triton",
|
||||
"can_use_mxfp8_swizzled": "quantization.mxfp8_swizzled_triton",
|
||||
"can_use_silu_mul_mxfp8": "quantization.mxfp8_swizzled_triton",
|
||||
"indexed_scale_shift_mxfp8_": "quantization.mxfp8_swizzled_triton",
|
||||
"mxfp8_quantize_swizzled": "quantization.mxfp8_swizzled_triton",
|
||||
"silu_mul_mxfp8": "quantization.mxfp8_swizzled_triton",
|
||||
"can_use_mxfp8_swizzled": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
|
||||
"can_use_silu_mul_mxfp8": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
|
||||
"indexed_scale_shift_mxfp8_": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
|
||||
"mxfp8_quantize_swizzled": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
|
||||
"silu_mul_mxfp8": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
|
||||
"can_use_usp_merge_heads": "layout.usp_relayout_jit",
|
||||
"usp_merge_heads": "layout.usp_relayout_jit",
|
||||
"build_inv_indices": "layout.varlen_pack_pad_triton",
|
||||
|
||||
@@ -7,7 +7,7 @@ import torch
|
||||
import triton # type: ignore
|
||||
import triton.language as tl # type: ignore
|
||||
|
||||
from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32
|
||||
from sglang.kernels.numerics import round_bf16_to_fp32
|
||||
|
||||
|
||||
@triton.jit
|
||||
|
||||
@@ -20,7 +20,7 @@ import torch
|
||||
import triton # type: ignore
|
||||
import triton.language as tl # type: ignore
|
||||
|
||||
from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32
|
||||
from sglang.kernels.numerics import round_bf16_to_fp32
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Shared infrastructure for the diffusion kernels -- no kernels of its own.
|
||||
|
||||
- ``numerics`` : rounding/opmath primitives the bit-exact kernels are built from
|
||||
- ``platform`` : device predicates and the Triton-vs-fallback selector
|
||||
- ``fallback_*``: pure-torch / NPU / MPS implementations for Triton-less devices
|
||||
"""
|
||||
|
||||
@@ -4,7 +4,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32
|
||||
from sglang.kernels.numerics import round_bf16_to_fp32
|
||||
|
||||
# Column tiles per row. One program per whole row padded to a power of two
|
||||
# masked off 37.5% of MiniMax-H3's 5376-wide rows; 2048-wide tiles run both
|
||||
|
||||
@@ -2,10 +2,10 @@ import torch
|
||||
import triton # type: ignore
|
||||
import triton.language as tl # type: ignore
|
||||
|
||||
from sglang.kernels.numerics import mul_rn_f32
|
||||
from sglang.kernels.ops.diffusion.common.fallback_torch import (
|
||||
fuse_scale_shift_kernel_native,
|
||||
)
|
||||
from sglang.kernels.ops.diffusion.common.numerics import mul_rn_f32
|
||||
from sglang.kernels.ops.diffusion.common.platform import (
|
||||
is_cuda,
|
||||
is_hip,
|
||||
|
||||
@@ -18,7 +18,7 @@ import torch
|
||||
import triton # type: ignore
|
||||
import triton.language as tl # type: ignore
|
||||
|
||||
from sglang.kernels.ops.diffusion.common.numerics import mul_rn_f32
|
||||
from sglang.kernels.numerics import mul_rn_f32
|
||||
|
||||
_MAX_BLOCK = 4096
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ import torch
|
||||
import triton # type: ignore
|
||||
import triton.language as tl # type: ignore
|
||||
|
||||
from sglang.kernels.ops.diffusion.common.numerics import (
|
||||
from sglang.kernels.numerics import (
|
||||
mul_rn_f32,
|
||||
round_bf16_to_fp32,
|
||||
rsqrt_approx_f32,
|
||||
|
||||
@@ -29,7 +29,7 @@ import torch
|
||||
import triton # type: ignore
|
||||
import triton.language as tl # type: ignore
|
||||
|
||||
from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32
|
||||
from sglang.kernels.numerics import round_bf16_to_fp32
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
|
||||
|
||||
@@ -182,6 +182,8 @@ _TRITON_KERNELS = [
|
||||
("awq_triton", "awq_dequantize_triton"),
|
||||
("awq_triton", "awq_gemm_triton"),
|
||||
("mxfp8_amd_gfx95", "mxfp8_e4m3_quantize"),
|
||||
("mxfp8_swizzled_triton", "swiglu_oai_mxfp8"),
|
||||
("mxfp8_swizzled_triton", "swiglu_oai_mxfp8_deepgemm"),
|
||||
]
|
||||
for _mod, _fn in _TRITON_KERNELS:
|
||||
register_kernel(
|
||||
|
||||
+94
-21
@@ -18,14 +18,14 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.diffusion.common.numerics import round_bf16_to_fp32
|
||||
from sglang.kernels.numerics import round_bf16_to_fp32
|
||||
|
||||
_E4M3 = torch.float8_e4m3fn
|
||||
|
||||
|
||||
def _scale_numel(rows: int, k: int) -> int:
|
||||
n_groups = k // 32
|
||||
return -(-rows // 128) * 128 * (-(-n_groups // 4) * 4)
|
||||
return triton.cdiv(rows, 128) * 128 * (triton.cdiv(n_groups, 4) * 4)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -96,6 +96,10 @@ def _silu_mul_mxfp8_kernel(
|
||||
n_groups,
|
||||
n_col_blocks,
|
||||
stride_row,
|
||||
dg_rows,
|
||||
ALPHA: tl.constexpr,
|
||||
LIMIT: tl.constexpr,
|
||||
DEEPGEMM: tl.constexpr,
|
||||
BLOCK_R: tl.constexpr,
|
||||
G: tl.constexpr,
|
||||
):
|
||||
@@ -109,10 +113,20 @@ def _silu_mul_mxfp8_kernel(
|
||||
base = x_ptr + r[:, None].to(tl.int64) * stride_row
|
||||
gate = tl.load(base + c[None, :], mask=mask, other=0.0).to(tl.float32)
|
||||
up = tl.load(base + hidden + c[None, :], mask=mask, other=0.0).to(tl.float32)
|
||||
act = (gate * tl.sigmoid(gate)).to(tl.bfloat16).to(tl.float32)
|
||||
prod = (act * up).to(tl.bfloat16).to(tl.float32)
|
||||
if ALPHA is None:
|
||||
act = (gate * tl.sigmoid(gate)).to(tl.bfloat16).to(tl.float32)
|
||||
prod = (act * up).to(tl.bfloat16).to(tl.float32)
|
||||
else:
|
||||
# the op order Inductor emits for swiglu_no_interleaved_with_alpha_and_limit
|
||||
gate = tl.minimum(gate, LIMIT, propagate_nan=tl.PropagateNan.ALL)
|
||||
up = tl.maximum(up, -LIMIT, propagate_nan=tl.PropagateNan.ALL)
|
||||
up = tl.minimum(up, LIMIT, propagate_nan=tl.PropagateNan.ALL)
|
||||
prod = gate * tl.sigmoid(gate * ALPHA) * (up + 1.0)
|
||||
prod = prod.to(tl.bfloat16).to(tl.float32)
|
||||
p3 = tl.reshape(prod, [BLOCK_R, G, 32])
|
||||
amax = tl.max(tl.abs(p3), axis=2)
|
||||
if DEEPGEMM:
|
||||
amax = tl.maximum(amax, 1e-10)
|
||||
sbyte, inv = _mx_e8m0_from_amax(amax)
|
||||
q = tl.reshape(p3 * inv[:, :, None], [BLOCK_R, G * 32])
|
||||
tl.store(
|
||||
@@ -120,12 +134,24 @@ def _silu_mul_mxfp8_kernel(
|
||||
q.to(tl.float8e4nv),
|
||||
mask=mask,
|
||||
)
|
||||
smask = rmask[:, None] & (g < n_groups)[None, :]
|
||||
tl.store(
|
||||
s_ptr + _mx_scale_offsets(r[:, None], g[None, :], n_col_blocks),
|
||||
sbyte.to(tl.uint8),
|
||||
mask=smask,
|
||||
)
|
||||
valid = rmask[:, None] & (g < n_groups)[None, :]
|
||||
if DEEPGEMM:
|
||||
# 4 bytes per int32 along K, MN-major, rows padded to 4
|
||||
soff = (
|
||||
(g[None, :] // 4).to(tl.int64) * dg_rows * 4
|
||||
+ r[:, None] * 4
|
||||
+ g[None, :] % 4
|
||||
)
|
||||
tl.store(s_ptr + soff, sbyte.to(tl.uint8), mask=valid)
|
||||
else:
|
||||
smask = (r < tl.cdiv(rows, 128) * 128)[:, None] & (g < n_col_blocks * 4)[
|
||||
None, :
|
||||
]
|
||||
tl.store(
|
||||
s_ptr + _mx_scale_offsets(r[:, None], g[None, :], n_col_blocks),
|
||||
tl.where(valid, sbyte, 0).to(tl.uint8),
|
||||
mask=smask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -194,10 +220,11 @@ def can_use_mxfp8_swizzled(x: torch.Tensor) -> bool:
|
||||
|
||||
|
||||
def _alloc(
|
||||
rows: int, k: int, device: torch.device
|
||||
rows: int, k: int, device: torch.device, zero_scales: bool = True
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
q = torch.empty(rows, k, dtype=_E4M3, device=device)
|
||||
s = torch.zeros(_scale_numel(rows, k), dtype=torch.uint8, device=device)
|
||||
alloc = torch.zeros if zero_scales else torch.empty
|
||||
s = alloc(_scale_numel(rows, k), dtype=torch.uint8, device=device)
|
||||
return q, s
|
||||
|
||||
|
||||
@@ -210,7 +237,7 @@ def mxfp8_quantize_swizzled(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor
|
||||
if rows == 0:
|
||||
return q, s
|
||||
n_groups = k // 32
|
||||
n_col_blocks = -(-n_groups // 4)
|
||||
n_col_blocks = triton.cdiv(n_groups, 4)
|
||||
block_r, g = 32, 8
|
||||
grid = (triton.cdiv(rows, block_r), triton.cdiv(n_groups, g))
|
||||
with torch.get_device_module().device(x.device):
|
||||
@@ -234,31 +261,52 @@ def can_use_silu_mul_mxfp8(hidden: torch.Tensor) -> bool:
|
||||
return can_use_mxfp8_swizzled(hidden) and (hidden.shape[-1] // 2) % 32 == 0
|
||||
|
||||
|
||||
def silu_mul_mxfp8(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""``hidden [rows, 2n]`` bf16 (gate | up) -> quantized ``silu(gate) * up``."""
|
||||
def _swiglu_mxfp8(
|
||||
hidden: torch.Tensor,
|
||||
alpha: float | None,
|
||||
limit: float | None,
|
||||
deepgemm: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if not can_use_silu_mul_mxfp8(hidden):
|
||||
raise ValueError(
|
||||
"expected a row-major bf16 CUDA [rows, 2 * n] tensor, n % 32 == 0"
|
||||
)
|
||||
rows, twice = hidden.shape
|
||||
n = twice // 2
|
||||
q, s = _alloc(rows, n, hidden.device)
|
||||
if deepgemm:
|
||||
if n % 128:
|
||||
raise ValueError(f"DeepGEMM MXFP8 scales need n % 128 == 0, got {n=}")
|
||||
q = torch.empty(rows, n, dtype=_E4M3, device=hidden.device)
|
||||
dg_rows = triton.cdiv(rows, 4) * 4
|
||||
base = torch.empty(n // 128, dg_rows, dtype=torch.int32, device=hidden.device)
|
||||
s = base.T[:rows]
|
||||
s_bytes = base.view(torch.uint8)
|
||||
padded_rows = rows
|
||||
else:
|
||||
q, s = _alloc(rows, n, hidden.device, zero_scales=False)
|
||||
s_bytes = s
|
||||
dg_rows = 0
|
||||
padded_rows = triton.cdiv(rows, 128) * 128
|
||||
if rows == 0:
|
||||
return q, s
|
||||
n_groups = n // 32
|
||||
n_col_blocks = -(-n_groups // 4)
|
||||
block_r, g = 16, 8
|
||||
grid = (triton.cdiv(rows, block_r), triton.cdiv(n_groups, g))
|
||||
n_col_blocks = triton.cdiv(n_groups, 4)
|
||||
block_r, g = (4 if rows <= 1024 else 16), 8
|
||||
grid = (triton.cdiv(padded_rows, block_r), triton.cdiv(n_groups, g))
|
||||
with torch.get_device_module().device(hidden.device):
|
||||
_silu_mul_mxfp8_kernel[grid](
|
||||
hidden,
|
||||
q,
|
||||
s,
|
||||
s_bytes,
|
||||
rows,
|
||||
n,
|
||||
n_groups,
|
||||
n_col_blocks,
|
||||
hidden.stride(0),
|
||||
dg_rows,
|
||||
ALPHA=alpha,
|
||||
LIMIT=limit,
|
||||
DEEPGEMM=deepgemm,
|
||||
BLOCK_R=block_r,
|
||||
G=g,
|
||||
num_warps=4,
|
||||
@@ -266,6 +314,29 @@ def silu_mul_mxfp8(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return q, s
|
||||
|
||||
|
||||
def silu_mul_mxfp8(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""``hidden [rows, 2n]`` bf16 (gate | up) -> quantized ``silu(gate) * up``."""
|
||||
return _swiglu_mxfp8(hidden, None, None)
|
||||
|
||||
|
||||
def swiglu_oai_mxfp8(
|
||||
hidden: torch.Tensor, alpha: float, limit: float
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""``hidden [rows, 2n]`` bf16 (gate | up) -> quantized SwiGLU-OAI,
|
||||
``clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``."""
|
||||
return _swiglu_mxfp8(hidden, alpha, limit)
|
||||
|
||||
|
||||
def swiglu_oai_mxfp8_deepgemm(
|
||||
hidden: torch.Tensor, alpha: float, limit: float
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Same values as ``swiglu_oai_mxfp8``; scales in DeepGEMM's layout, the
|
||||
``sglang_per_token_group_quant_fp8(..., 32, column_major_scales=True,
|
||||
scale_tma_aligned=True, scale_ue8m0=True)`` output: int32 ``[rows, n / 128]``,
|
||||
four UE8M0 bytes each, MN-major with rows padded to 4."""
|
||||
return _swiglu_mxfp8(hidden, alpha, limit, deepgemm=True)
|
||||
|
||||
|
||||
def indexed_scale_shift_mxfp8_(
|
||||
x: torch.Tensor,
|
||||
shift: torch.Tensor,
|
||||
@@ -285,7 +356,7 @@ def indexed_scale_shift_mxfp8_(
|
||||
if rows == 0:
|
||||
return (x if keep_bf16 else None), q, s
|
||||
n_groups = hidden_size // 32
|
||||
n_col_blocks = -(-n_groups // 4)
|
||||
n_col_blocks = triton.cdiv(n_groups, 4)
|
||||
block_n = triton.next_power_of_2(hidden_size)
|
||||
with torch.get_device_module().device(x.device):
|
||||
_indexed_scale_shift_mxfp8_kernel[(rows,)](
|
||||
@@ -315,4 +386,6 @@ __all__ = [
|
||||
"indexed_scale_shift_mxfp8_",
|
||||
"mxfp8_quantize_swizzled",
|
||||
"silu_mul_mxfp8",
|
||||
"swiglu_oai_mxfp8",
|
||||
"swiglu_oai_mxfp8_deepgemm",
|
||||
]
|
||||
@@ -878,13 +878,18 @@ def _deepgemm_w8a8_mxfp8_linear_with_fallback(
|
||||
sglang_per_token_group_quant_fp8,
|
||||
)
|
||||
|
||||
assert input_scale is None
|
||||
output_dtype = input.dtype
|
||||
pre_quantized = input_scale is not None
|
||||
output_dtype = torch.bfloat16 if pre_quantized else input.dtype
|
||||
|
||||
shape_supported = weight.shape[0] % 64 == 0 and weight.shape[1] % 128 == 0
|
||||
dtype_supported = output_dtype == torch.bfloat16
|
||||
|
||||
if not (shape_supported and dtype_supported):
|
||||
if pre_quantized:
|
||||
raise ValueError(
|
||||
"Pre-quantized DeepGEMM input requires N % 64 == 0 and K % 128 == 0; "
|
||||
"its packed scales cannot be used by the FlashInfer fallback."
|
||||
)
|
||||
if weight_scale_swizzled is None:
|
||||
raise RuntimeError(
|
||||
f"DeepGEMM cannot serve this MXFP8 GEMM ({shape_supported=}, "
|
||||
@@ -902,13 +907,32 @@ def _deepgemm_w8a8_mxfp8_linear_with_fallback(
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
output_shape = [*input.shape[:-1], weight.shape[0]]
|
||||
|
||||
q_input, x_scale = sglang_per_token_group_quant_fp8(
|
||||
input_2d,
|
||||
32,
|
||||
column_major_scales=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0,
|
||||
scale_tma_aligned=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0,
|
||||
scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0,
|
||||
)
|
||||
if pre_quantized:
|
||||
m, k = input_2d.shape
|
||||
if not (
|
||||
deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
and input.ndim == 2
|
||||
and input.is_contiguous()
|
||||
and input.dtype == torch.float8_e4m3fn
|
||||
and input_scale.dtype == torch.int32
|
||||
and input_scale.device == input.device
|
||||
and input_scale.shape == (m, k // 128)
|
||||
and input_scale.stride() == (1, ceil_align(m, 4))
|
||||
and k == weight.shape[1]
|
||||
):
|
||||
raise ValueError(
|
||||
"Expected 2D contiguous FP8 input with packed, MN-major, "
|
||||
"TMA-aligned UE8M0 scales for DeepGEMM MXFP8."
|
||||
)
|
||||
q_input, x_scale = input_2d, input_scale
|
||||
else:
|
||||
q_input, x_scale = sglang_per_token_group_quant_fp8(
|
||||
input_2d,
|
||||
32,
|
||||
column_major_scales=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0,
|
||||
scale_tma_aligned=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0,
|
||||
scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0,
|
||||
)
|
||||
|
||||
# weight_scale format is set per-backend by _process_mxfp8_linear_weight_scale
|
||||
# (int32 packed TMA-aligned on Blackwell, float32 on Hopper); NOT uint8 — Triton form is routed to the fallback above.
|
||||
|
||||
@@ -14,3 +14,40 @@ class Mxfp8SwizzledInput(NamedTuple):
|
||||
|
||||
data: torch.Tensor
|
||||
scales: torch.Tensor
|
||||
|
||||
|
||||
def accepts_mxfp8_swizzled_input(linear) -> bool:
|
||||
"""Whether a linear can consume the explicit 128x4 producer contract."""
|
||||
qm = getattr(linear, "quant_method", None)
|
||||
backend = getattr(qm, "mxfp8_dense_backend", None)
|
||||
return bool(
|
||||
backend is not None
|
||||
and not getattr(qm, "use_marlin", False)
|
||||
and (
|
||||
getattr(qm, "use_mxfp8", False)
|
||||
or (
|
||||
getattr(qm, "block_fp8_as_mxfp8", False)
|
||||
and getattr(linear, "block_fp8_mxfp8_ready", False)
|
||||
)
|
||||
)
|
||||
and (backend.is_flashinfer_cutlass() or backend.is_flashinfer_cutedsl())
|
||||
)
|
||||
|
||||
|
||||
def accepts_mxfp8_deepgemm_input(linear) -> bool:
|
||||
"""Whether a linear takes a ``(q, scale)`` pair with DeepGEMM's packed UE8M0 scales."""
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
|
||||
qm = getattr(linear, "quant_method", None)
|
||||
backend = getattr(qm, "mxfp8_dense_backend", None)
|
||||
weight = getattr(linear, "weight", None)
|
||||
return bool(
|
||||
backend is not None
|
||||
and backend.is_deep_gemm()
|
||||
and getattr(qm, "use_mxfp8", False)
|
||||
and not getattr(qm, "use_marlin", False)
|
||||
and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
and weight is not None
|
||||
and weight.shape[0] % 64 == 0
|
||||
and weight.shape[1] % 128 == 0
|
||||
)
|
||||
|
||||
@@ -23,6 +23,11 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.kernels.ops.quantization.mxfp8_swizzled_triton import (
|
||||
can_use_silu_mul_mxfp8,
|
||||
swiglu_oai_mxfp8,
|
||||
swiglu_oai_mxfp8_deepgemm,
|
||||
)
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
|
||||
from sglang.srt.configs.model_config import (
|
||||
get_minimax_sparse_disable_value_layer_ids,
|
||||
@@ -64,6 +69,11 @@ from sglang.srt.layers.moe.utils import (
|
||||
is_shared_experts_fusion_disabled,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.mxfp8_input import (
|
||||
Mxfp8SwizzledInput,
|
||||
accepts_mxfp8_deepgemm_input,
|
||||
accepts_mxfp8_swizzled_input,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.utils import PPMissingLayer
|
||||
@@ -90,7 +100,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
from sglang.srt.models.deepseek_common.utils import tiny_router_gemm_max_tokens
|
||||
from sglang.srt.models.minimax_m2 import MiniMaxM2RMSNormTP
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream
|
||||
from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
get_device_sm,
|
||||
@@ -286,6 +296,7 @@ class MiniMaxM3MLP(nn.Module):
|
||||
super().__init__()
|
||||
hidden_size = config.hidden_size
|
||||
hidden_act = config.hidden_act
|
||||
self.fuse_swiglu_oai_mxfp8 = hidden_act == "swigluoai"
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
@@ -307,9 +318,11 @@ class MiniMaxM3MLP(nn.Module):
|
||||
if hidden_act == "silu":
|
||||
self.act_fn = SiluAndMul()
|
||||
elif hidden_act == "swigluoai":
|
||||
self.swiglu_alpha = float(config.swiglu_alpha)
|
||||
self.swiglu_limit = float(config.swiglu_limit)
|
||||
if _is_npu:
|
||||
self.act_fn = lambda x: self._swigluoai_fused(
|
||||
x, config.swiglu_alpha, config.swiglu_limit
|
||||
x, self.swiglu_alpha, self.swiglu_limit
|
||||
)
|
||||
else:
|
||||
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import (
|
||||
@@ -317,20 +330,40 @@ class MiniMaxM3MLP(nn.Module):
|
||||
)
|
||||
|
||||
self.act_fn = lambda x: swiglu_no_interleaved_with_alpha_and_limit(
|
||||
x, config.swiglu_alpha, config.swiglu_limit
|
||||
x, self.swiglu_alpha, self.swiglu_limit
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
|
||||
)
|
||||
|
||||
def _swiglu_oai_mxfp8(self, gate_up: torch.Tensor):
|
||||
if not (
|
||||
self.fuse_swiglu_oai_mxfp8
|
||||
and gate_up.shape[0] > 0
|
||||
and not get_forward().sp_active
|
||||
and can_use_silu_mul_mxfp8(gate_up)
|
||||
):
|
||||
return None
|
||||
if accepts_mxfp8_swizzled_input(self.down_proj):
|
||||
return Mxfp8SwizzledInput(
|
||||
*swiglu_oai_mxfp8(gate_up, self.swiglu_alpha, self.swiglu_limit)
|
||||
)
|
||||
if accepts_mxfp8_deepgemm_input(self.down_proj):
|
||||
return swiglu_oai_mxfp8_deepgemm(
|
||||
gate_up, self.swiglu_alpha, self.swiglu_limit
|
||||
)
|
||||
return None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x,
|
||||
forward_batch: Optional[ForwardBatch] = None,
|
||||
):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x = self._swiglu_oai_mxfp8(gate_up)
|
||||
if x is None:
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
Reference in New Issue
Block a user