[AMD][V4.1][*/N] Fuse MXFP8 activation quant into producer kernels on gfx950 (#42055)
Co-authored-by: wunhuang <wunhuang@amd.com>
This commit is contained in:
@@ -68,9 +68,12 @@ def _silu_and_mul_clamp_kernel(
|
||||
)
|
||||
|
||||
|
||||
def silu_and_mul_clamp_fp8_grid_supported(intermediate: int) -> bool:
|
||||
"""The fp8-grid epilogue needs whole 32-wide groups inside one program."""
|
||||
return intermediate % 32 == 0 and intermediate <= 1024
|
||||
def silu_and_mul_clamp_fp8_grid_supported(
|
||||
intermediate: int, multi_block: bool = False
|
||||
) -> bool:
|
||||
"""The fp8-grid epilogue needs whole 32-wide groups inside one program; 1024-wide programs
|
||||
keep that for any multiple of 32 (multi_block)."""
|
||||
return intermediate % 32 == 0 and (multi_block or intermediate <= 1024)
|
||||
|
||||
|
||||
def silu_and_mul_clamp_triton(
|
||||
@@ -87,8 +90,8 @@ def silu_and_mul_clamp_triton(
|
||||
inter_size = N // 2
|
||||
fp8_grid = fp8_grid or emit_fp8
|
||||
if fp8_grid:
|
||||
assert silu_and_mul_clamp_fp8_grid_supported(inter_size), (
|
||||
f"inter_size {inter_size}: the fp8-grid epilogue needs a multiple of 32 up to 1024"
|
||||
assert silu_and_mul_clamp_fp8_grid_supported(inter_size, multi_block=True), (
|
||||
f"inter_size {inter_size}: the fp8-grid epilogue needs a multiple of 32"
|
||||
)
|
||||
out = torch.empty(
|
||||
(M, inter_size),
|
||||
|
||||
@@ -11,7 +11,10 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import fp8_grid_round
|
||||
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import (
|
||||
fp8_grid_quant,
|
||||
fp8_grid_round,
|
||||
)
|
||||
|
||||
# 16 x 32 x 512, 2 warps: the fastest N tile holding whole 32-groups; nonkdim 16 is aiter's MFMA
|
||||
_BLOCK_M, _BLOCK_N, _BLOCK_K = 16, 32, 512
|
||||
@@ -50,11 +53,15 @@ def _batched_gemm_bf16_fp8_grid_kernel(
|
||||
stride_cb,
|
||||
stride_cm,
|
||||
stride_cn,
|
||||
scale_ptr,
|
||||
stride_sb,
|
||||
stride_sm,
|
||||
BLOCK_SIZE_M: tl.constexpr,
|
||||
BLOCK_SIZE_N: tl.constexpr,
|
||||
BLOCK_SIZE_K: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
FP8_GRID: tl.constexpr,
|
||||
EMIT_FP8: tl.constexpr,
|
||||
cache_modifier: tl.constexpr,
|
||||
num_warps: tl.constexpr,
|
||||
num_stages: tl.constexpr,
|
||||
@@ -117,12 +124,21 @@ def _batched_gemm_bf16_fp8_grid_kernel(
|
||||
a_ptrs += BLOCK_SIZE_K * stride_ak
|
||||
b_ptrs += BLOCK_SIZE_K * stride_bk
|
||||
|
||||
c = accumulator.to(c_ptr.type.element_ty)
|
||||
if FP8_GRID:
|
||||
# the consumer's fake-quant on the bf16-rounded output: one ue8m0 group per 32 N elements
|
||||
xg = tl.reshape(c.to(tl.float32), (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32))
|
||||
c = tl.reshape(fp8_grid_round(xg), (BLOCK_SIZE_M, BLOCK_SIZE_N))
|
||||
c = c.to(c_ptr.type.element_ty)
|
||||
if EMIT_FP8:
|
||||
# the consumer's own quant of the bf16-rounded output: fp8 codes + one ue8m0 per 32 N
|
||||
xg = tl.reshape(
|
||||
accumulator.to(tl.bfloat16).to(tl.float32),
|
||||
(BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32),
|
||||
)
|
||||
q8, e8 = fp8_grid_quant(xg)
|
||||
c = tl.reshape(q8, (BLOCK_SIZE_M, BLOCK_SIZE_N))
|
||||
else:
|
||||
c = accumulator.to(c_ptr.type.element_ty)
|
||||
if FP8_GRID:
|
||||
# the consumer's fake-quant on the bf16-rounded output: one ue8m0 group per 32 N elements
|
||||
xg = tl.reshape(c.to(tl.float32), (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32))
|
||||
c = tl.reshape(fp8_grid_round(xg), (BLOCK_SIZE_M, BLOCK_SIZE_N))
|
||||
c = c.to(c_ptr.type.element_ty)
|
||||
|
||||
offs_cm = tl.cast(pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M), tl.int64)
|
||||
offs_cn = tl.cast(pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N), tl.int64)
|
||||
@@ -134,6 +150,16 @@ def _batched_gemm_bf16_fp8_grid_kernel(
|
||||
)
|
||||
c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
|
||||
tl.store(c_ptrs, c, mask=c_mask)
|
||||
if EMIT_FP8:
|
||||
offs_sn = pid_n * (BLOCK_SIZE_N // 32) + tl.arange(0, BLOCK_SIZE_N // 32)
|
||||
tl.store(
|
||||
scale_ptr
|
||||
+ stride_sb * batch_id
|
||||
+ stride_sm * offs_cm[:, None]
|
||||
+ offs_sn[None, :],
|
||||
tl.reshape(e8, (BLOCK_SIZE_M, BLOCK_SIZE_N // 32)).to(tl.uint8),
|
||||
mask=(offs_cm[:, None] < M) & (offs_sn[None, :] < N // 32),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -208,10 +234,14 @@ def _batched_gemm_split_k_reduce_kernel(
|
||||
N,
|
||||
stride_cb,
|
||||
stride_cm,
|
||||
scale_ptr,
|
||||
stride_sb,
|
||||
stride_sm,
|
||||
BLOCK_SIZE_M: tl.constexpr,
|
||||
BLOCK_SIZE_N: tl.constexpr,
|
||||
SPLIT_K: tl.constexpr,
|
||||
FP8_GRID: tl.constexpr,
|
||||
EMIT_FP8: tl.constexpr,
|
||||
):
|
||||
"""Grid (G, row tiles x N tiles): sums the partials of one output tile in split order,
|
||||
rounds to bf16 and, with FP8_GRID, onto the consumer's fp8 grid."""
|
||||
@@ -231,11 +261,19 @@ def _batched_gemm_split_k_reduce_kernel(
|
||||
+ (tile + tl.arange(0, BLOCK_SIZE_M))[:, None] * N
|
||||
+ offs_n[None, :]
|
||||
)
|
||||
c = acc.to(c_ptr.type.element_ty)
|
||||
if FP8_GRID:
|
||||
xg = tl.reshape(c.to(tl.float32), (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32))
|
||||
c = tl.reshape(fp8_grid_round(xg), (BLOCK_SIZE_M, BLOCK_SIZE_N))
|
||||
c = c.to(c_ptr.type.element_ty)
|
||||
if EMIT_FP8:
|
||||
xg = tl.reshape(
|
||||
acc.to(tl.bfloat16).to(tl.float32),
|
||||
(BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32),
|
||||
)
|
||||
q8, e8 = fp8_grid_quant(xg)
|
||||
c = tl.reshape(q8, (BLOCK_SIZE_M, BLOCK_SIZE_N))
|
||||
else:
|
||||
c = acc.to(c_ptr.type.element_ty)
|
||||
if FP8_GRID:
|
||||
xg = tl.reshape(c.to(tl.float32), (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32))
|
||||
c = tl.reshape(fp8_grid_round(xg), (BLOCK_SIZE_M, BLOCK_SIZE_N))
|
||||
c = c.to(c_ptr.type.element_ty)
|
||||
c_ptrs = (
|
||||
c_ptr
|
||||
+ stride_cb * batch_id
|
||||
@@ -243,6 +281,16 @@ def _batched_gemm_split_k_reduce_kernel(
|
||||
+ offs_n[None, :]
|
||||
)
|
||||
tl.store(c_ptrs, c, mask=(offs_m[:, None] < M) & (offs_n[None, :] < N))
|
||||
if EMIT_FP8:
|
||||
offs_sn = pid_n * (BLOCK_SIZE_N // 32) + tl.arange(0, BLOCK_SIZE_N // 32)
|
||||
tl.store(
|
||||
scale_ptr
|
||||
+ stride_sb * batch_id
|
||||
+ stride_sm * tl.cast(offs_m, tl.int64)[:, None]
|
||||
+ offs_sn[None, :],
|
||||
tl.reshape(e8, (BLOCK_SIZE_M, BLOCK_SIZE_N // 32)).to(tl.uint8),
|
||||
mask=(offs_m[:, None] < M) & (offs_sn[None, :] < N // 32),
|
||||
)
|
||||
|
||||
|
||||
def _split_k_applies(T: int, D: int, R: int) -> bool:
|
||||
@@ -255,7 +303,11 @@ def _split_k_applies(T: int, D: int, R: int) -> bool:
|
||||
|
||||
|
||||
def _batched_gemm_split_k(
|
||||
x: torch.Tensor, w: torch.Tensor, out: torch.Tensor, fp8_grid: bool
|
||||
x: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
fp8_grid: bool,
|
||||
scale: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
T, G, D = x.shape
|
||||
R = w.shape[1]
|
||||
@@ -293,10 +345,14 @@ def _batched_gemm_split_k(
|
||||
R,
|
||||
R,
|
||||
G * R,
|
||||
scale if scale is not None else out,
|
||||
R // 32,
|
||||
G * R // 32,
|
||||
BLOCK_SIZE_M=_BLOCK_M,
|
||||
BLOCK_SIZE_N=_BLOCK_N,
|
||||
SPLIT_K=_SPLIT_K,
|
||||
FP8_GRID=fp8_grid,
|
||||
EMIT_FP8=scale is not None,
|
||||
num_warps=_NUM_WARPS,
|
||||
)
|
||||
|
||||
@@ -306,26 +362,38 @@ def batched_gemm_bf16_fp8_grid(
|
||||
w: torch.Tensor,
|
||||
fp8_grid: bool = True,
|
||||
split_k: Optional[bool] = None,
|
||||
) -> torch.Tensor:
|
||||
emit_fp8: bool = False,
|
||||
):
|
||||
"""x [T, G, D] bf16, w [G, R, D] bf16 -> [T, G * R] bf16 with out[t, g*R:(g+1)*R] =
|
||||
x[t, g] @ w[g]^T, on the fp8 grid when fp8_grid; split_k=None picks the regime by shape,
|
||||
False keeps the single launch at every T (deterministic inference)."""
|
||||
False keeps the single launch at every T (deterministic inference).
|
||||
emit_fp8 returns (fp8 e4m3 [T, G * R], ue8m0 uint8 [T, G * R // 32]) instead."""
|
||||
assert x.dim() == 3 and w.dim() == 3, (x.shape, w.shape)
|
||||
T, G, D = x.shape
|
||||
assert w.shape[0] == G and w.shape[2] == D, (x.shape, w.shape)
|
||||
R = w.shape[1]
|
||||
assert x.dtype == torch.bfloat16 and w.dtype == torch.bfloat16
|
||||
assert x.stride(2) == 1 and w.is_contiguous()
|
||||
assert not fp8_grid or R % 32 == 0, R
|
||||
out = torch.empty((T, G * R), dtype=torch.bfloat16, device=x.device)
|
||||
assert not (fp8_grid or emit_fp8) or R % 32 == 0, R
|
||||
out = torch.empty(
|
||||
(T, G * R),
|
||||
dtype=torch.float8_e4m3fn if emit_fp8 else torch.bfloat16,
|
||||
device=x.device,
|
||||
)
|
||||
scale = (
|
||||
torch.empty((T, G * R // 32), dtype=torch.uint8, device=x.device)
|
||||
if emit_fp8
|
||||
else None
|
||||
)
|
||||
result = (out, scale) if emit_fp8 else out
|
||||
if T == 0:
|
||||
return out
|
||||
return result
|
||||
if split_k is None:
|
||||
split_k = _split_k_applies(T, D, R)
|
||||
if split_k:
|
||||
assert _split_k_applies(T, D, R), (T, D, R)
|
||||
_batched_gemm_split_k(x, w, out, fp8_grid)
|
||||
return out
|
||||
_batched_gemm_split_k(x, w, out, fp8_grid, scale)
|
||||
return result
|
||||
block_m, block_n, block_k, num_warps = _tile_config(T)
|
||||
grid = (G, triton.cdiv(T, block_m) * triton.cdiv(R, block_n))
|
||||
_batched_gemm_bf16_fp8_grid_kernel[grid](
|
||||
@@ -344,15 +412,19 @@ def batched_gemm_bf16_fp8_grid(
|
||||
R,
|
||||
G * R,
|
||||
1,
|
||||
scale if scale is not None else out,
|
||||
R // 32,
|
||||
G * R // 32,
|
||||
BLOCK_SIZE_M=block_m,
|
||||
BLOCK_SIZE_N=block_n,
|
||||
BLOCK_SIZE_K=block_k,
|
||||
EVEN_K=(D % block_k == 0),
|
||||
FP8_GRID=fp8_grid,
|
||||
EMIT_FP8=emit_fp8,
|
||||
cache_modifier=_CACHE_MODIFIER,
|
||||
num_warps=num_warps,
|
||||
num_stages=_NUM_STAGES,
|
||||
waves_per_eu=_WAVES_PER_EU,
|
||||
matrix_instr_nonkdim=_MFMA_NONKDIM,
|
||||
)
|
||||
return out
|
||||
return result
|
||||
|
||||
@@ -1592,6 +1592,12 @@ class Envs:
|
||||
# DSpark draft block on the HIP radix backend: build the attention metadata inside the
|
||||
# draft CUDA graph from the raw inputs instead of eagerly before every replay.
|
||||
SGLANG_HIP_DSPARK_DRAFT_RAW_METADATA = EnvBool(_default_hip)
|
||||
# gfx950 MXFP8 dense routes (aiter group32 / native): the producer emits fp8 + ue8m0 for its
|
||||
# consumer instead of bf16 plus a separate quant launch -- the shared expert's SwiGLU for
|
||||
# down_proj, the wo_a GEMM for wo_b, and the FFN norm for the shared expert's gate_up.
|
||||
SGLANG_HIP_SHARED_ACT_MXFP8 = EnvBool(_default_hip)
|
||||
SGLANG_HIP_WO_A_MXFP8 = EnvBool(_default_hip)
|
||||
SGLANG_HIP_FFN_NORM_MXFP8 = EnvBool(_default_hip)
|
||||
|
||||
# cache, GEMM, and distributed
|
||||
SGLANG_OPT_FP8_WO_A_GEMM = EnvBool(True)
|
||||
|
||||
@@ -10,6 +10,7 @@ from sglang.kernels.ops.activation.silu_and_mul_clamp_hip import (
|
||||
silu_and_mul_clamp_fp8_grid_supported,
|
||||
silu_and_mul_clamp_triton,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
|
||||
|
||||
|
||||
@@ -33,7 +34,9 @@ def resolve_fused_clamp_route(mlp, half_width: int) -> None:
|
||||
quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native()
|
||||
or quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_aiter()
|
||||
)
|
||||
and silu_and_mul_clamp_fp8_grid_supported(half_width)
|
||||
and silu_and_mul_clamp_fp8_grid_supported(
|
||||
half_width, multi_block=envs.SGLANG_HIP_SHARED_ACT_MXFP8.get()
|
||||
)
|
||||
)
|
||||
# the native and aiter MXFP8 routes take fp8 + ue8m0 straight from the epilogue
|
||||
mlp._hip_act_native_consumer = bool(
|
||||
|
||||
@@ -165,12 +165,36 @@ def post_attention_norm(layer, x: torch.Tensor, coefficients=None) -> torch.Tens
|
||||
if coefficients is not None:
|
||||
coefficients.materialize()
|
||||
return norm(x)
|
||||
if _ffn_norm_emits_mxfp8(layer, x.shape[0]):
|
||||
x_quant, out = rmsnorm_with_sinkhorn(
|
||||
x, norm.weight.data, norm.variance_epsilon, coefficients, emit_fp8=True
|
||||
)
|
||||
# the shared expert's gate_up takes it in place of its own quant (_forward_shared_experts)
|
||||
out._hip_mxfp8_operand = x_quant
|
||||
return out
|
||||
_, out = rmsnorm_with_sinkhorn(
|
||||
x, norm.weight.data, norm.variance_epsilon, coefficients, fake_quant=False
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
_FFN_NORM_EMIT_MXFP8 = envs.SGLANG_HIP_FFN_NORM_MXFP8.get()
|
||||
|
||||
|
||||
def _ffn_norm_emits_mxfp8(layer, num_tokens: int) -> bool:
|
||||
"""SGLANG_HIP_FFN_NORM_MXFP8: whether the FFN norm launch also emits fp8 + ue8m0 for the
|
||||
shared expert's gate_up (its route consumes them at this token count)."""
|
||||
if not _FFN_NORM_EMIT_MXFP8:
|
||||
return False
|
||||
consumer = getattr(layer, "_ffn_shared_mxfp8_consumer", None)
|
||||
if consumer is None:
|
||||
shared = getattr(getattr(layer, "mlp", None), "shared_experts", None)
|
||||
consumer = layer._ffn_shared_mxfp8_consumer = _mxfp8_consumer(
|
||||
getattr(shared, "gate_up_proj", None)
|
||||
)
|
||||
return consumer
|
||||
|
||||
|
||||
def live_rows(activation, num_tokens: int):
|
||||
"""The first num_tokens rows of a break input; the gfx950 fused q_norm
|
||||
hands the indexer its Fp8GridActivation wrapper, whose rows live in .x."""
|
||||
@@ -196,15 +220,37 @@ def wo_b_takes_fp8_grid(attn) -> bool:
|
||||
return attn._wo_b_fp8_grid_operand
|
||||
|
||||
|
||||
def wo_a_fp8_grid_matmul(o: torch.Tensor, wo_a: torch.Tensor, fp8_grid: bool):
|
||||
_WO_A_EMIT_MXFP8 = envs.SGLANG_HIP_WO_A_MXFP8.get()
|
||||
|
||||
|
||||
def wo_b_emits_mxfp8(attn, num_tokens: int) -> bool:
|
||||
"""SGLANG_HIP_WO_A_MXFP8: whether the wo_a GEMM hands wo_b fp8 + ue8m0 from its epilogue
|
||||
(wo_b's route consumes them at this token count) instead of bf16 plus a separate quant."""
|
||||
if not _WO_A_EMIT_MXFP8 or _wo_a_fp8_grid_gemm is None:
|
||||
return False
|
||||
consumer = getattr(attn, "_wo_b_mxfp8_consumer", None)
|
||||
if consumer is None:
|
||||
consumer = attn._wo_b_mxfp8_consumer = _mxfp8_consumer(attn.wo_b)
|
||||
return consumer
|
||||
|
||||
|
||||
def wo_a_fp8_grid_matmul(
|
||||
o: torch.Tensor, wo_a: torch.Tensor, fp8_grid: bool, emit_fp8: bool = False
|
||||
):
|
||||
"""o [T, G, D] @ wo_a [G, R, D]^T through the gfx950 fork of the aiter batched
|
||||
GEMM: bf16 [T, G, R], or with fp8_grid rounded onto wo_b's fp8 grid as
|
||||
an Fp8GridActivation [T, G * R]; None when the fork did not import."""
|
||||
an Fp8GridActivation [T, G * R], or with emit_fp8 an Mxfp8Activation [T, G * R];
|
||||
None when the fork did not import."""
|
||||
if _wo_a_fp8_grid_gemm is None:
|
||||
return None
|
||||
# the split-K regime ends at 64 rows, so a request's verify and decode rows would
|
||||
# take different reduction orders; deterministic inference keeps the single chain
|
||||
split_k = False if get_exec().deterministic.enable_deterministic_inference else None
|
||||
if emit_fp8:
|
||||
q, scale = _wo_a_fp8_grid_gemm(
|
||||
o, wo_a, fp8_grid=False, split_k=split_k, emit_fp8=True
|
||||
)
|
||||
return Mxfp8Activation(q, scale)
|
||||
y = _wo_a_fp8_grid_gemm(o, wo_a, fp8_grid=fp8_grid, split_k=split_k)
|
||||
if fp8_grid:
|
||||
return Fp8GridActivation(y)
|
||||
|
||||
@@ -28,6 +28,7 @@ from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
live_rows = gfx95_dense.live_rows
|
||||
wo_a_fp8_grid_matmul = gfx95_dense.wo_a_fp8_grid_matmul
|
||||
wo_b_emits_mxfp8 = gfx95_dense.wo_b_emits_mxfp8
|
||||
|
||||
_is_gfx95_supported = is_gfx95_supported()
|
||||
|
||||
|
||||
@@ -259,6 +259,7 @@ _moe_quant_once_logged = False
|
||||
_enable_pcg_dsv2_dual_stream = (
|
||||
_is_cuda and envs.SGLANG_ENABLE_PCG_DSV2_DUAL_STREAM.get()
|
||||
)
|
||||
_hip_shared_act_mxfp8 = _is_hip and envs.SGLANG_HIP_SHARED_ACT_MXFP8.get()
|
||||
|
||||
|
||||
class DeepseekV2MLP(nn.Module):
|
||||
@@ -411,7 +412,16 @@ class DeepseekV2MLP(nn.Module):
|
||||
if self.use_fused_clamp_act_mul and not self._fused_clamp_fp8_checked:
|
||||
_hip_act.resolve_fused_clamp_route(self, gate_up.shape[-1] // 2)
|
||||
|
||||
if self.use_fused_clamp_act_mul and self.swiglu_limit is not None:
|
||||
# SGLANG_HIP_SHARED_ACT_MXFP8: an MXFP8 down_proj takes fp8 + ue8m0 straight from the Triton
|
||||
# epilogue instead of aiter's bf16 output plus a separate quant launch.
|
||||
if (
|
||||
self.use_fused_clamp_act_mul
|
||||
and self.swiglu_limit is not None
|
||||
and _hip_shared_act_mxfp8
|
||||
and self._hip_act_fp8_grid
|
||||
):
|
||||
x = _hip_act.silu_and_mul_clamp(self, gate_up)
|
||||
elif self.use_fused_clamp_act_mul and self.swiglu_limit is not None:
|
||||
from aiter.ops.triton.fusions.fused_clamp_act_mul import (
|
||||
fused_clamp_act_mul,
|
||||
)
|
||||
@@ -1733,6 +1743,9 @@ class DeepseekV2MoE(nn.Module):
|
||||
pre_quant_input: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
):
|
||||
if (hidden_states.shape[0] > 0) and (self.num_fused_shared_experts == 0):
|
||||
if pre_quant_input is None and _is_hip:
|
||||
# SGLANG_HIP_FFN_NORM_MXFP8: the FFN norm launch's fp8 + ue8m0 of these rows
|
||||
pre_quant_input = getattr(hidden_states, "_hip_mxfp8_operand", None)
|
||||
if pre_quant_input is not None:
|
||||
# SGLANG_OPT_MOE_QUANT_ONCE: (q, s) rows may be padded to a
|
||||
# multiple of 4; the padded rows flow through the MLP (all ops
|
||||
|
||||
@@ -462,7 +462,8 @@ def _apply_wo_a_bf16_matmul(
|
||||
is_prefill: bool = False,
|
||||
fast_path: bool = False,
|
||||
fp8_grid: bool = False,
|
||||
) -> torch.Tensor | Mxfp8SwizzledInput:
|
||||
emit_fp8: bool = False,
|
||||
) -> torch.Tensor | Mxfp8SwizzledInput | Fp8GridActivation | Mxfp8Activation:
|
||||
# o [T, G, D] @ wo_a [G, R, D] -> [T, G, R]; the fast paths below are gated
|
||||
# on the exact validated TP4 shapes and write token-major output directly.
|
||||
global _wo_a_aiter_batched_gemm_disabled
|
||||
@@ -528,7 +529,7 @@ def _apply_wo_a_bf16_matmul(
|
||||
return result
|
||||
# the 16-row decode kernels also serve V4.1's target-verify rows
|
||||
if (is_decode or is_target_verify) and hip_fast_path:
|
||||
y = _hip.wo_a_fp8_grid_matmul(o, wo_a, fp8_grid)
|
||||
y = _hip.wo_a_fp8_grid_matmul(o, wo_a, fp8_grid, emit_fp8=emit_fp8)
|
||||
if y is not None:
|
||||
return y
|
||||
if (
|
||||
@@ -669,6 +670,10 @@ def _apply_gguf_grouped_wo_a(
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import (
|
||||
Fp8GridActivation,
|
||||
Mxfp8Activation,
|
||||
)
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
DeepseekV4AttnBackend,
|
||||
LateLayerTail,
|
||||
@@ -2579,6 +2584,8 @@ class MQALayer(MqaAttentionBase):
|
||||
fast_path=self.is_dsv41,
|
||||
fuse_mxfp8_quant=fuse_mxfp8_quant,
|
||||
fp8_grid=_is_hip and _hip.wo_a_emits_fp8_grid(self),
|
||||
emit_fp8=_is_hip
|
||||
and _hip.wo_b_emits_mxfp8(self, o.shape[0]),
|
||||
)
|
||||
else:
|
||||
o = _apply_gguf_grouped_wo_a(
|
||||
|
||||
Reference in New Issue
Block a user