diff --git a/python/sglang/kernels/ops/layernorm/rmsnorm_fp8_e8m0.py b/python/sglang/kernels/ops/layernorm/rmsnorm_fp8_e8m0.py new file mode 100644 index 0000000000..bd281d3a09 --- /dev/null +++ b/python/sglang/kernels/ops/layernorm/rmsnorm_fp8_e8m0.py @@ -0,0 +1,98 @@ +"""Fused RMSNorm + per-token 1x128 fp8 quant with power-of-two (ue8m0) scales. + +The e8m0 counterpart of aiter's ``fused_rms_fp8_group_quant(..., +output_unquantized_inp1=True, transpose_scale=True)``: one launch emits the bf16 +normed row (for the readers that want it unquantized) and the fp8 operand of an +e8m0 mxscale GEMM. The scale is ``ceil_pow2(amax / 448)`` (aiter's default +RoundUp / DeepSeek-V4 reference ``act_quant(scale_fmt="ue8m0")``), stored as one +e8m0 byte per group in column-major order: a contiguous ``[K/128, M]`` buffer +viewed as ``[M, K/128]``, the layout ``per_group_quant_hip(transpose_scale=True, +scale_type=fp8_e8m0)`` emits and aiter's mxscale bpreshuffle GEMM reads. +""" + +from typing import Tuple + +import torch +import triton +import triton.language as tl + +_FP8_MAX = 448.0 + + +@triton.jit +def _rmsnorm_fp8_e8m0_kernel( + x_ptr, + w_ptr, + y_ptr, + q_ptr, + s_ptr, + M, + N, + x_stride, + y_stride, + q_stride, + eps, + BLOCK_N: tl.constexpr, + GROUP: tl.constexpr, + NUM_G: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + offs = tl.arange(0, BLOCK_N) + mask = offs < N + x = tl.load(x_ptr + row * x_stride + offs, mask=mask, other=0.0).to(tl.float32) + rstd = tl.math.rsqrt(tl.sum(x * x, axis=0) / N + eps) + w = tl.load(w_ptr + offs, mask=mask, other=0.0).to(tl.float32) + y = (x * rstd * w).to(y_ptr.dtype.element_ty) + tl.store(y_ptr + row * y_stride + offs, y, mask=mask) + + # Quantize the rounded bf16 row, as a separate quant of the norm output would. + yg = tl.reshape(y.to(tl.float32), (NUM_G, GROUP)) + amax = tl.max(tl.abs(yg), axis=1) + bits = (amax * (1.0 / 448.0)).to(tl.int32, bitcast=True) + e = (bits >> 23) & 0xFF + e = tl.where(((bits & 0x7FFFFF) != 0) & (e < 0xFF), e + 1, e) + inv = tl.where(e > 0, 1.0 / (e << 23).to(tl.float32, bitcast=True), 0.0) + q = tl.clamp(yg * inv[:, None], -448.0, 448.0).to(q_ptr.dtype.element_ty) + q_offs = tl.reshape(offs, (NUM_G, GROUP)) + tl.store( + q_ptr + row * q_stride + q_offs, + q, + mask=tl.reshape(mask, (NUM_G, GROUP)), + ) + g = tl.arange(0, NUM_G) + tl.store(s_ptr + g * M + row, e.to(tl.uint8), mask=g * GROUP < N) + + +def rmsnorm_fp8_e8m0_quant( + x: torch.Tensor, weight: torch.Tensor, eps: float, group_size: int = 128 +) -> Tuple[Tuple[torch.Tensor, torch.Tensor], torch.Tensor]: + """Returns ``((q_fp8 [M, N], scale_e8m0 [M, N/128] column-major bytes), y [M, N])``.""" + assert x.dim() == 2 and x.stride(-1) == 1 + m, n = x.shape + assert n % group_size == 0 + num_g = n // group_size + y = torch.empty((m, n), dtype=x.dtype, device=x.device) + q = torch.empty((m, n), dtype=torch.float8_e4m3fn, device=x.device) + s = torch.empty((num_g, m), dtype=torch.float8_e8m0fnu, device=x.device) + if m > 0: + num_g_pow2 = triton.next_power_of_2(num_g) + block_n = num_g_pow2 * group_size + num_warps = 4 if block_n <= 2048 else (8 if block_n <= 4096 else 16) + _rmsnorm_fp8_e8m0_kernel[(m,)]( + x, + weight, + y, + q, + s.view(torch.uint8), + m, + n, + x.stride(0), + y.stride(0), + q.stride(0), + eps, + BLOCK_N=block_n, + GROUP=group_size, + NUM_G=num_g_pow2, + num_warps=num_warps, + ) + return (q, s.view(m, num_g)), y diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index ff8f8ddf8a..8772906b2e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1613,6 +1613,15 @@ class Envs: # only way to ask; on separate-KV it is the reverse -- --kv-cache-dtype # picks the buffer dtype and this switch is inert. SGLANG_DSV4_UNIFIED_KV_FP8 = EnvBool(False) + # ROCm gfx950, DeepSeek-V4 128x128 block-fp8 dense linears: run the GEMM as + # aiter's e8m0 mxscale bpreshuffle (flydsl) instead of the fp32-scale + # CK/asm/triton bpreshuffle. The weight bytes are shared; only an e8m0 copy + # of the (power-of-two) weight scale is added at load, and activations are + # quantized with power-of-two (ue8m0) scales. Restricted to the (N, K) + # shapes in SGLANG_DSV4_DENSE_MXSCALE_SHAPES ("N:K,N:K"; empty = built-in + # whitelist of shapes that win in the microbench, "all" = every shape). + SGLANG_DSV4_DENSE_MXSCALE = EnvBool(False) + SGLANG_DSV4_DENSE_MXSCALE_SHAPES = EnvStr("") # fp8 unified_kv decode/verify on aiter's FlyDSL v4 nm decode when the # installed aiter ships it; 0 keeps the asm kernel. SGLANG_DSV4_FLYDSL_DECODE = EnvBool(True) diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index 5542b41666..9d0c3e0116 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -56,11 +56,13 @@ from sglang.srt.layers.quantization.fp8_utils import ( _deepgemm_block_fp8_supported, _use_aiter_bpreshuffle_gfx95, apply_fp8_linear, + block_fp8_scale_to_e8m0_128, block_fp8_scale_to_mxfp8_e8m0, can_auto_enable_marlin_fp8, can_serve_block_fp8_as_mxfp8, cutlass_fp8_supported, deepgemm_w8a8_block_fp8_linear_with_fallback, + dense_mxscale_wanted, dispatch_block_fp8_mxfp8_linear, dispatch_w8a8_block_fp8_linear, dispatch_w8a8_mxfp8_linear, @@ -183,6 +185,7 @@ if _use_aiter: ACTIVATION_SCHEMES = ["static", "dynamic"] logger = logging.getLogger(__name__) +_DENSE_MXSCALE_LOGGED: set = set() DSV4_DEQUANT_FP4_TABLE = torch.tensor( [ @@ -902,6 +905,7 @@ class Fp8LinearMethod(LinearMethodBase): # instead of silently reading a permuted weight. layer.aiter_bpreshuffled = True layer.weight.is_shuffled = True + self._maybe_prepare_dense_mxscale(layer) if ( is_xpu() @@ -926,6 +930,36 @@ class Fp8LinearMethod(LinearMethodBase): with torch.no_grad(): layer.weight_scale_inv.set_(scale_reordered) + def _maybe_prepare_dense_mxscale(self, layer: Module) -> None: + """SGLANG_DSV4_DENSE_MXSCALE: give a (16, 16)-preshuffled 128x128 block-fp8 + weight an e8m0 copy of its scale, which routes aiter_w8a8_block_fp8_linear + onto aiter's mxscale bpreshuffle GEMM. Lossless only for power-of-two + scales (ue8m0 checkpoints); any other layer stays on the fp32 GEMM. The + weight bytes are shared, the copy costs N*K/16384 bytes.""" + if self.weight_block_size != [128, 128] or self.convert_mxfp8_to_block: + return + if layer.weight.dtype != torch.float8_e4m3fn: + return + n, k = layer.weight.shape + if not dense_mxscale_wanted(n, k): + return + scale = layer.weight_scale_inv.data + if tuple(scale.shape) != (n // 128, k // 128) or n % 128 or k % 128: + return + e8m0 = block_fp8_scale_to_e8m0_128(scale) + if e8m0 is None: + logger.warning( + "Dense mxscale: %s keeps the fp32 GEMM (scales are not powers of two)", + getattr(layer, "prefix", tuple(layer.weight.shape)), + ) + return + layer.weight._mxscale_w_scale = e8m0 + if (n, k) not in _DENSE_MXSCALE_LOGGED: + _DENSE_MXSCALE_LOGGED.add((n, k)) + logger.info( + "Dense mxscale (e8m0 bpreshuffle GEMM) enabled for N=%d K=%d", n, k + ) + def _prepare_block_fp8_as_mxfp8(self, layer: Module) -> None: layer.block_fp8_mxfp8_ready = False if getattr(layer, "keep_plain_weight_layout", False): diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index cfcd6e1088..39acc95de2 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -121,6 +121,121 @@ def set_force_ck_w8a8(enabled: bool = True) -> None: _FORCE_CK_W8A8 = enabled +# DeepSeek-V4 dense 128x128 block-fp8 linears on aiter's e8m0 mxscale bpreshuffle +# GEMM (SGLANG_DSV4_DENSE_MXSCALE). DeepseekV4 enables it via +# set_dense_mxscale(True) before its weights load, like set_force_ck_w8a8. +_DENSE_MXSCALE: bool = False + +# (N, K) where the e8m0 GEMM (with aiter's dsv4 mxscale tuned rows) beats the +# fp32-scale bpreshuffle GEMM (incl. aiter #6004/#6029 tuned rows) on gfx950, +# cold-weight, over the decode M range. See dsv4_perf_opt reports/dense_gemm.md. +# (512, 7168) (DSpark draft wkv) is untuned on the e8m0 side and loses at M>=96. +DSV4_DENSE_MXSCALE_DEFAULT_SHAPES = frozenset( + { + (2048, 7168), # wqkv_a + (1536, 7168), # wq_a (unfused: DSpark draft) + (8192, 1536), # wq_b TP8, indexer wq_b + (16384, 1536), # wq_b TP4 + (65536, 1536), # wq_b attn-TP1 (DP attention) + (7168, 2048), # wo_b TP8 + (7168, 4096), # wo_b TP4 + (7168, 16384), # wo_b attn-TP1 (DP attention) + (6144, 7168), # shared expert w13, TP1 + (7168, 3072), # shared expert w2, TP1 + (768, 7168), # shared expert w13, TP8 + (7168, 384), # shared expert w2, TP8 + (7168, 21504), # DSpark main_proj + } +) + + +@lru_cache(maxsize=1) +def aiter_has_dense_mxscale() -> bool: + """Whether the installed aiter dispatches e8m0 block scales in + gemm_a8w8_blockscale_bpreshuffle. Older aiter takes the e8m0 bytes as fp32 + scales and silently returns wrong results.""" + try: + from aiter.ops.gemm_op_a8w8 import get_mxscale_bpreshuffle_config # noqa: F401 + except Exception: + return False + return True + + +def set_dense_mxscale(enabled: bool = True) -> None: + global _DENSE_MXSCALE + if enabled and not aiter_has_dense_mxscale(): + logger.warning( + "SGLANG_DSV4_DENSE_MXSCALE is set but the installed aiter has no e8m0 " + "mxscale bpreshuffle GEMM; keeping the fp32-scale GEMM." + ) + enabled = False + _DENSE_MXSCALE = enabled + + +@lru_cache(maxsize=1) +def _dense_mxscale_shapes() -> Optional[frozenset]: + """The (N, K) whitelist, or None for every shape.""" + spec = envs.SGLANG_DSV4_DENSE_MXSCALE_SHAPES.get().strip() + if not spec: + return DSV4_DENSE_MXSCALE_DEFAULT_SHAPES + if spec.lower() == "all": + return None + shapes = set() + for item in spec.split(","): + n, k = item.strip().split(":") + shapes.add((int(n), int(k))) + return frozenset(shapes) + + +def dense_mxscale_wanted(n: int, k: int) -> bool: + if not (_DENSE_MXSCALE and _use_aiter_bpreshuffle_gfx95): + return False + shapes = _dense_mxscale_shapes() + return shapes is None or (n, k) in shapes + + +def block_fp8_scale_to_e8m0_128(scale: torch.Tensor) -> Optional[torch.Tensor]: + """fp32 128x128 block scales [N/128, K/128] -> e8m0 bytes of the same shape, + or None if any scale is not a positive normal power of two (not lossless).""" + s = scale.detach().float().contiguous() + bits = s.view(torch.int32) + exp = (bits >> 23) & 0xFF + ok = (s > 0) & ((bits & 0x7FFFFF) == 0) & (exp > 0) & (exp < 0xFF) + if not bool(ok.all()): + return None + return exp.to(torch.uint8).view(torch.float8_e8m0fnu).contiguous() + + +def is_e8m0_scale(scale: Optional[torch.Tensor]) -> bool: + return scale is not None and scale.dtype == torch.float8_e8m0fnu + + +def dense_mxscale_ready(*linears) -> bool: + """Whether every given linear consumes e8m0 (mxscale) activations, i.e. a + shared producer of their fp8 operand should emit ue8m0 scales.""" + if not _DENSE_MXSCALE: + return False + for linear in linears: + if linear is None: + continue + weight = getattr(linear, "weight", None) + if getattr(weight, "_mxscale_w_scale", None) is None: + return False + return True + + +def quant_fp8_e8m0_128(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Per-token 1x128 fp8 quant with ue8m0 scales, the scale bytes column-major + ([K/128, M] storage viewed as [M, K/128]) as the mxscale GEMM reads them.""" + return per_group_quant_hip( + x, + quant_dtype=aiter.dtypes.fp8, + group_size=128, + transpose_scale=True, + scale_type=torch.float8_e8m0fnu, + ) + + def materialize_bpreshuffle_fp8_scale(scale: torch.Tensor) -> torch.Tensor: """Materialize the physical scale layout consumed by gfx95 bpreshuffle GEMM.""" return scale.t().contiguous().t() if scale.dim() == 2 else scale @@ -235,6 +350,8 @@ if _use_aiter: ) aiter_per1x128_quant = get_hip_quant(aiter.QuantType.per_1x128) + from aiter.ops.quant import per_group_quant_hip + from sglang.kernels.ops.gemm.smallm_fp8_gfx950 import ( smallm_fp8_gemm, smallm_fp8_gemm_supported, @@ -1474,6 +1591,30 @@ def aiter_w8a8_block_fp8_linear( ) return out.to(input.dtype).view(*output_shape) + # DeepSeek-V4 e8m0 mxscale route (SGLANG_DSV4_DENSE_MXSCALE): same + # (16, 16)-preshuffled weight, e8m0 copy of its scale. A producer that + # already emitted an fp32-scale operand keeps the fp32 GEMM below. + mx_w_scale = getattr(weight, "_mxscale_w_scale", None) + if is_e8m0_scale(input_scale) or (mx_w_scale is not None and input_scale is None): + if mx_w_scale is None: + raise RuntimeError( + "e8m0-scaled fp8 activation reached a linear without an e8m0 " + f"weight scale (weight {tuple(weight.shape)})" + ) + out_dtype = torch.bfloat16 if input_scale is not None else input_2d.dtype + if input_2d.shape[0] == 0: + return input_2d.new_empty(output_shape, dtype=out_dtype) + if input_scale is None: + q_input, x_scale = quant_fp8_e8m0_128(input_2d) + else: + q_input, x_scale = input_2d, input_scale + output = gemm_a8w8_blockscale_bpreshuffle( + q_input, weight, x_scale, mx_w_scale, dtype=torch.bfloat16 + ) + if bias is not None: + output += bias + return output.to(dtype=out_dtype).view(*output_shape) + n, k = weight.shape if _use_aiter_bpreshuffle_gfx95: diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py index f0a871fe5b..82345b7b12 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py @@ -180,6 +180,26 @@ def hc_post(cfg, x, residual, post, comb) -> Optional[torch.Tensor]: return mhc_post_split_h(x, residual, post, comb, block_size=2048) +def _x_quant_e8m0(layer) -> bool: + """Whether input_layernorm's fp8 operand goes out with ue8m0 scales: only when + every dense projection reading it (wqkv_a, or wq_a + wkv) runs the e8m0 mxscale + GEMM (SGLANG_DSV4_DENSE_MXSCALE). Resolved on first use, after weight load.""" + cached = layer.__dict__.get("_x_quant_e8m0") + if cached is None: + from sglang.srt.layers.quantization.fp8_utils import dense_mxscale_ready + + attn = layer.self_attn + if getattr(attn, "fuse_wqa_wkv", False): + consumers = (getattr(attn, "wqkv_a", None),) + else: + consumers = (getattr(attn, "wq_a", None), getattr(attn, "wkv", None)) + cached = all(c is not None for c in consumers) and dense_mxscale_ready( + *consumers + ) + layer.__dict__["_x_quant_e8m0"] = cached + return cached + + def input_norm( layer, hidden_states: torch.Tensor, @@ -194,7 +214,10 @@ def input_norm( if coefficients is not None: coefficients.materialize() x_quant, hidden_states = fused_rmsnorm_fp8_quant( - hidden_states, layer.input_layernorm.weight, layer.rms_norm_eps + hidden_states, + layer.input_layernorm.weight, + layer.rms_norm_eps, + e8m0=_x_quant_e8m0(layer), ) return hidden_states, x_quant if layer.fused_rmsnorm_fake_quant: diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 01fea9977f..4f097ae353 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -551,7 +551,14 @@ def _apply_wo_a_bf16_matmul( return torch.einsum("tgd,grd->tgr", o, wo_a) -def _fused_rmsnorm_fp8_quant(hidden_states, weight, eps): +def _fused_rmsnorm_fp8_quant(hidden_states, weight, eps, e8m0: bool = False): + if e8m0: + # Consumers run the e8m0 mxscale GEMM (SGLANG_DSV4_DENSE_MXSCALE). + from sglang.kernels.ops.layernorm.rmsnorm_fp8_e8m0 import ( + rmsnorm_fp8_e8m0_quant, + ) + + return rmsnorm_fp8_e8m0_quant(hidden_states, weight, eps) x_quant, x_bf16, _, _ = fused_rms_fp8_group_quant( hidden_states, weight, @@ -1126,6 +1133,22 @@ class MqaAttentionBase(nn.Module): self._attn_sink_local = sink return self._attn_sink_local[:kernel_num_heads] + def _q_lora_quant_e8m0(self, with_indexer: bool) -> bool: + """Whether the fused q_norm quant feeds wq_b (and the indexer's wq_b, which + reuses it) with ue8m0 scales: only when each consumer runs the e8m0 mxscale + GEMM. Resolved on first use, after the weights are processed.""" + key = "_q_lora_e8m0_idx" if with_indexer else "_q_lora_e8m0" + cached = self.__dict__.get(key) + if cached is None: + from sglang.srt.layers.quantization.fp8_utils import dense_mxscale_ready + + indexer = getattr(self, "indexer", None) if with_indexer else None + cached = dense_mxscale_ready( + self.wq_b, getattr(indexer, "wq_b", None) if indexer else None + ) + self.__dict__[key] = cached + return cached + @contextmanager def maybe_use_decode_attn_tp(self, forward_batch: ForwardBatch): ctx = get_cp_decode_attn_tp_ctx() @@ -1897,6 +1920,7 @@ class MQALayer(MqaAttentionBase): q_lora, self.q_norm.weight, self.q_norm.variance_epsilon, + e8m0=self._q_lora_quant_e8m0(with_indexer=False), ) q, _ = self.wq_b(q_for_wqb) else: @@ -2036,6 +2060,7 @@ class MQALayer(MqaAttentionBase): q_lora, self.q_norm.weight, self.q_norm.variance_epsilon, + e8m0=self._q_lora_quant_e8m0(with_indexer=True), ) q, _ = self.wq_b(q_for_wqb) else: @@ -4251,10 +4276,15 @@ class DeepseekV4ForCausalLM(nn.Module): # Module-level toggles default OFF; flipped True here for DSV4 if _is_hip: from sglang.kernels.ops.attention.deepseek_v4_rope import set_batched_rope - from sglang.srt.layers.quantization.fp8_utils import set_force_ck_w8a8 + from sglang.srt.layers.quantization.fp8_utils import ( + set_dense_mxscale, + set_force_ck_w8a8, + ) set_force_ck_w8a8(True) set_batched_rope(True) + if envs.SGLANG_DSV4_DENSE_MXSCALE.get(): + set_dense_mxscale(True) self.config = config self.tp_size = get_parallel().tp_size self.quant_config = quant_config diff --git a/test/registered/kernels/ops/quantization/test_dsv4_dense_mxscale_gfx95.py b/test/registered/kernels/ops/quantization/test_dsv4_dense_mxscale_gfx95.py new file mode 100644 index 0000000000..24775bca40 --- /dev/null +++ b/test/registered/kernels/ops/quantization/test_dsv4_dense_mxscale_gfx95.py @@ -0,0 +1,264 @@ +"""DeepSeek-V4 dense 128x128 block-fp8 linears on aiter's e8m0 mxscale bpreshuffle +GEMM (SGLANG_DSV4_DENSE_MXSCALE) against the fp32-scale bpreshuffle route and an +fp32 dequant reference; the fused RMSNorm + ue8m0 quant producer against torch and +aiter's per_group_quant_hip.""" + +import unittest + +import torch + +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd-mi35x") + +# (N, K) of the DSv4-Pro dense fp8 projections: wqkv_a, wq_b TP8 / indexer wq_b, +# wo_b TP8, wo_b attn-TP1, wq_b attn-TP1, shared-expert down TP1, DSpark main_proj. +SHAPES = [ + (2048, 7168), + (8192, 1536), + (7168, 2048), + (7168, 16384), + (65536, 1536), + (7168, 3072), + (7168, 21504), +] +MS = [1, 7, 64, 129, 256, 2048] + + +def _ue8m0_ref_quant(x: torch.Tensor): + """Per-token 1x128 fp8 quant, scale = ceil_pow2(amax / 448) as fp32.""" + m, k = x.shape + g = x.float().view(m, k // 128, 128) + s = g.abs().amax(-1, keepdim=True) * torch.tensor(1.0 / 448.0, dtype=torch.float32) + bits = s.view(torch.int32) + e = (bits >> 23) & 0xFF + e = torch.where(((bits & 0x7FFFFF) != 0) & (e < 0xFF), e + 1, e) + scale = (e << 23).view(torch.float32) + inv = torch.where(e > 0, 1.0 / scale, torch.zeros_like(scale)) + q = (g * inv).clamp(-448, 448).to(torch.float8_e4m3fn).view(m, k) + return q, e.view(m, k // 128).to(torch.uint8), scale.view(m, k // 128) + + +def _aiter_has_dense_mxscale() -> bool: + if not (is_hip() and is_gfx95_supported()): + return False + from sglang.srt.layers.quantization.fp8_utils import aiter_has_dense_mxscale + + return aiter_has_dense_mxscale() + + +@unittest.skipUnless(_aiter_has_dense_mxscale(), "gfx950 aiter mxscale GEMM") +class TestDsv4DenseMxscale(CustomTestCase): + @classmethod + def setUpClass(cls): + from sglang.srt.layers.quantization import fp8_utils + + fp8_utils.set_force_ck_w8a8(True) + fp8_utils.set_dense_mxscale(True) + cls.fu = fp8_utils + + @classmethod + def tearDownClass(cls): + cls.fu.set_dense_mxscale(False) + + def _weight(self, n, k, seed): + from aiter.ops.shuffle import shuffle_weight + + g = torch.Generator(device="cuda").manual_seed(seed) + w = (torch.randn(n, k, device="cuda", generator=g) * 0.5).to( + torch.float8_e4m3fn + ) + exp = torch.randint(-12, -4, (n // 128, k // 128), device="cuda", generator=g) + scale = torch.exp2(exp.float()).contiguous() + w_deq = ( + w.float().view(n // 128, 128, k // 128, 128) * scale[:, None, :, None] + ).view(n, k) + return shuffle_weight(w, (16, 16)), scale, w_deq + + def test_scale_to_e8m0_is_lossless_and_rejects_non_pow2(self): + s = torch.exp2(torch.randint(-30, 10, (64, 12), device="cuda").float()) + e = self.fu.block_fp8_scale_to_e8m0_128(s) + self.assertEqual(e.dtype, torch.float8_e8m0fnu) + back = torch.exp2(e.view(torch.uint8).float() - 127) + torch.testing.assert_close(back, s, rtol=0, atol=0) + s2 = s.clone() + s2[3, 4] *= 1.5 + self.assertIsNone(self.fu.block_fp8_scale_to_e8m0_128(s2)) + self.assertIsNone(self.fu.block_fp8_scale_to_e8m0_128(torch.zeros_like(s))) + + def test_rmsnorm_fp8_e8m0_producer(self): + from aiter import dtypes + from aiter.ops.quant import per_group_quant_hip + + from sglang.kernels.ops.layernorm.rmsnorm_fp8_e8m0 import ( + rmsnorm_fp8_e8m0_quant, + ) + + for k in (7168, 1536): + for m in (1, 5, 96, 1024): + with self.subTest(k=k, m=m): + torch.manual_seed(m + k) + x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * 3 + x[0, :256] = 0 # an all-zero 128-group: scale byte 0, q 0 + w = torch.rand(k, device="cuda", dtype=torch.bfloat16) + 0.5 + (q, s), y = rmsnorm_fp8_e8m0_quant(x, w, 1e-6) + xf = x.float() + y_ref = ( + xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + 1e-6) + ) * w.float() + torch.testing.assert_close( + y.float(), + y_ref.to(torch.bfloat16).float(), + atol=2e-2, + rtol=1e-2, + ) + self.assertEqual(tuple(s.shape), (m, k // 128)) + self.assertEqual(s.dtype, torch.float8_e8m0fnu) + # column-major bytes: [K/128, M] storage + s_rm = s.view(torch.uint8).view(k // 128, m).t() + q_ref, e_ref, _ = _ue8m0_ref_quant(y) + self.assertTrue(torch.equal(s_rm, e_ref)) + mism = (q.view(torch.uint8) != q_ref.view(torch.uint8)).float() + self.assertLess(mism.mean().item(), 1e-3) + # same bytes as aiter's own e8m0 quant of the bf16 norm output + q_a, s_a = per_group_quant_hip( + y, + quant_dtype=dtypes.fp8, + group_size=128, + transpose_scale=True, + scale_type=dtypes.fp8_e8m0, + ) + # (aiter writes its own byte for an all-zero group; its q is 0 too) + live = e_ref != 0 + s_a_rm = s_a.view(torch.uint8).view(k // 128, m).t() + self.assertTrue(torch.equal(s_a_rm[live], s_rm[live])) + mism = (q_a.view(torch.uint8) != q.view(torch.uint8)).float() + self.assertLess(mism.mean().item(), 1e-3) + + def test_linear_matches_fp32_route(self): + fu = self.fu + for n, k in SHAPES: + wsh, s32, w_deq = self._weight(n, k, seed=n + k) + mx = fu.block_fp8_scale_to_e8m0_128(s32) + self.assertIsNotNone(mx) + for m in MS: + with self.subTest(n=n, k=k, m=m): + torch.manual_seed(m) + x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) + # fp32 route (current path) + y_fp32 = fu.aiter_w8a8_block_fp8_linear(x, wsh, [128, 128], s32) + # e8m0 route, bf16 input (the linear quantizes) + wsh._mxscale_w_scale = mx + try: + y_mx = fu.aiter_w8a8_block_fp8_linear(x, wsh, [128, 128], s32) + # e8m0 route, pre-quantized ue8m0 operand (fused producer) + q, sc = fu.quant_fp8_e8m0_128(x) + y_mx_pre = fu.aiter_w8a8_block_fp8_linear( + q, wsh, [128, 128], s32, input_scale=sc + ) + finally: + del wsh._mxscale_w_scale + # exact reference of the e8m0 operands + q_ref, _, a_scale = _ue8m0_ref_quant(x) + a_deq = ( + q_ref.float().view(m, k // 128, 128) * a_scale[..., None] + ).view(m, k) + ref_mx = a_deq @ w_deq.t() + ref_bf16 = x.float() @ w_deq.t() + nrm = ref_bf16.norm().clamp_min(1e-6) + err_mx = ((y_mx.float() - ref_mx).norm() / nrm).item() + self.assertLess(err_mx, 1e-2) + torch.testing.assert_close(y_mx_pre, y_mx, rtol=0, atol=0) + # both routes sit at the same distance from the bf16 product + e_fp32 = ((y_fp32.float() - ref_bf16).norm() / nrm).item() + e_mx = ((y_mx.float() - ref_bf16).norm() / nrm).item() + self.assertLess(e_mx, 1.25 * e_fp32 + 2e-3) + self.assertLess( + ((y_mx.float() - y_fp32.float()).norm() / nrm).item(), 6e-2 + ) + self.assertTrue(torch.isfinite(y_mx).all()) + + def test_route_selection(self): + fu = self.fu + wsh, s32, _ = self._weight(2048, 7168, seed=1) + x = torch.randn(8, 7168, device="cuda", dtype=torch.bfloat16) + q, sc = fu.quant_fp8_e8m0_128(x) + # an e8m0 operand into a linear without an e8m0 weight scale is an error + with self.assertRaises(RuntimeError): + fu.aiter_w8a8_block_fp8_linear(q, wsh, [128, 128], s32, input_scale=sc) + # an fp32 operand into an e8m0 linear keeps the fp32 GEMM + wsh._mxscale_w_scale = fu.block_fp8_scale_to_e8m0_128(s32) + try: + from sglang.srt.layers.quantization.fp8_utils import ( + aiter_per1x128_quant, + view_aiter_fused_rms_transposed_fp8_scale, + ) + + q32, s32x = aiter_per1x128_quant( + x, quant_dtype=torch.float8_e4m3fn, transpose_scale=True + ) + s32x = view_aiter_fused_rms_transposed_fp8_scale(s32x) + y = fu.aiter_w8a8_block_fp8_linear( + q32, wsh, [128, 128], s32, input_scale=s32x + ) + del wsh._mxscale_w_scale + y_ref = fu.aiter_w8a8_block_fp8_linear( + q32, wsh, [128, 128], s32, input_scale=s32x + ) + torch.testing.assert_close(y, y_ref, rtol=0, atol=0) + finally: + if hasattr(wsh, "_mxscale_w_scale"): + del wsh._mxscale_w_scale + + def test_shape_whitelist_and_ready(self): + fu = self.fu + self.assertTrue(fu.dense_mxscale_wanted(2048, 7168)) + self.assertTrue(fu.dense_mxscale_wanted(65536, 1536)) + self.assertFalse(fu.dense_mxscale_wanted(512, 7168)) + + class _L: + def __init__(self, mx): + self.weight = torch.empty(1) + if mx: + self.weight._mxscale_w_scale = torch.empty(1) + + self.assertTrue(fu.dense_mxscale_ready(_L(True), None, _L(True))) + self.assertFalse(fu.dense_mxscale_ready(_L(True), _L(False))) + + def test_process_weights_installs_e8m0_scale(self): + from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod + + method = Fp8LinearMethod.__new__(Fp8LinearMethod) + method.weight_block_size = [128, 128] + method.convert_mxfp8_to_block = False + for (n, k), pow2, want in ( + ((8192, 1536), True, True), + ((512, 7168), True, False), # not whitelisted + ((7168, 2048), False, False), # non power-of-two scales + ): + with self.subTest(n=n, k=k, pow2=pow2): + layer = torch.nn.Module() + layer.weight = torch.nn.Parameter( + torch.zeros(n, k, device="cuda", dtype=torch.float8_e4m3fn), + requires_grad=False, + ) + s = torch.full((n // 128, k // 128), 0.25, device="cuda") + if not pow2: + s[0, 0] = 0.3 + layer.weight_scale_inv = torch.nn.Parameter(s, requires_grad=False) + method._maybe_prepare_dense_mxscale(layer) + got = getattr(layer.weight, "_mxscale_w_scale", None) + self.assertEqual(got is not None, want) + if want: + self.assertTrue( + torch.equal( + got.view(torch.uint8), + torch.full_like(s, 125, dtype=torch.uint8), + ) + ) + + +if __name__ == "__main__": + unittest.main()