diff --git a/python/pyproject.toml b/python/pyproject.toml index 768ceb52ba..12e8dfcbb0 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -77,7 +77,7 @@ dependencies = [ "sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers. "setproctitle", "sgl-deep-ep==0.1.2.post1", - "sgl-deep-gemm==0.2.1.post1", + "sgl-deep-gemm==0.2.2", "sglang-kernel==0.4.9", "smg-grpc-servicer>=0.9.0", "soundfile==0.13.1", diff --git a/python/sglang/kernels/ops/gemm/fp8_kernel.py b/python/sglang/kernels/ops/gemm/fp8_kernel.py index 46234dc6eb..ff32551d33 100644 --- a/python/sglang/kernels/ops/gemm/fp8_kernel.py +++ b/python/sglang/kernels/ops/gemm/fp8_kernel.py @@ -33,8 +33,11 @@ def deep_gemm_fp8_fp8_bf16_nt( B: torch.Tensor, Bs: torch.Tensor, C: torch.Tensor, + block_n: int = 128, + block_k: int = 128, ) -> None: - deep_gemm_wrapper.gemm_nt_f8f8bf16((A, As), (B, Bs), C) + recipe = (1, block_n, block_k) if (block_n, block_k) != (128, 128) else None + deep_gemm_wrapper.gemm_nt_f8f8bf16((A, As), (B, Bs), C, recipe=recipe) @register_custom_op(mutates_args=["C"]) @@ -746,7 +749,7 @@ def w8a8_block_fp8_matmul_deepgemm( # Deepgemm only supports output tensor type as bfloat16 assert C.dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM - deep_gemm_fp8_fp8_bf16_nt(A, As, B, Bs, C) + deep_gemm_fp8_fp8_bf16_nt(A, As, B, Bs, C, block_size[0], block_size[1]) return C diff --git a/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py b/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py index 5caf3ea83c..1a45727db3 100644 --- a/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py +++ b/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py @@ -5,6 +5,7 @@ import os import time from contextlib import contextmanager, nullcontext from enum import IntEnum, auto +from functools import partial from pathlib import Path from typing import Dict, List, Tuple @@ -115,6 +116,7 @@ class DeepGemmKernelType(IntEnum): GROUPED_GEMM_NT_BF16_MASKED = auto() GROUPED_GEMM_NT_BF16_CONTIG = auto() GEMM_NT_F8F8BF16 = auto() + GEMM_NT_F8F8BF16_BLOCK32 = auto() GEMM_NT_BF16BF16F32 = auto() @@ -269,6 +271,9 @@ class _BaseWarmupExecutor: def create(kernel_type: DeepGemmKernelType, **kwargs): return { DeepGemmKernelType.GEMM_NT_F8F8BF16: _NormalWarmupExecutor, + DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32: partial( + _NormalWarmupExecutor, block_size=32 + ), DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_CONTIG: _GroupedContWarmupExecutor, DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_MASKED: _GroupedMaskedWarmupExecutor, DeepGemmKernelType.GEMM_NT_BF16BF16F32: _BF16F32WarmupExecutor, @@ -282,8 +287,17 @@ class _BaseWarmupExecutor: ) -> float: # FP32 input scales remain live while DeepGEMM converts their layout. _GB = 1 << 30 - lhs_scales = max_m * ceil_div(k, _BLOCK_SIZE) * 4 - rhs_scales = ceil_div(n, _BLOCK_SIZE) * ceil_div(k, _BLOCK_SIZE) * 4 + is_dense_fp8 = kernel_type in ( + DeepGemmKernelType.GEMM_NT_F8F8BF16, + DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32, + ) + block_size = ( + 32 + if kernel_type == DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32 + else _BLOCK_SIZE + ) + lhs_scales = max_m * ceil_div(k, block_size) * 4 + rhs_scales = ceil_div(n, block_size) * ceil_div(k, block_size) * 4 scale_workspace = _get_fp8_scale_workspace( max_m, n, @@ -293,11 +307,10 @@ class _BaseWarmupExecutor: if kernel_type == DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_MASKED else 1 ), - rhs_groups=( - 1 if kernel_type == DeepGemmKernelType.GEMM_NT_F8F8BF16 else num_groups - ), + rhs_groups=1 if is_dense_fp8 else num_groups, + block_size=block_size, ) - if kernel_type == DeepGemmKernelType.GEMM_NT_F8F8BF16: + if is_dense_fp8: return ( max_m * k + n * k @@ -358,9 +371,9 @@ def _packed_scale_workspace(rows: int, cols: int, groups: int) -> tuple[int, int def _get_fp8_scale_workspace( - max_m: int, n: int, k: int, lhs_groups: int, rhs_groups: int + max_m: int, n: int, k: int, lhs_groups: int, rhs_groups: int, block_size: int = 128 ) -> int: - scale_k = ceil_div(k, _BLOCK_SIZE) + scale_k = ceil_div(k, block_size) if DEEPGEMM_NEED_TMA_ALIGNED_SCALES: return lhs_groups * ceil_align(max_m, 4) * scale_k * 4 if DEEPGEMM_SCALE_UE8M0: @@ -373,22 +386,22 @@ def _get_fp8_scale_workspace( return 0 -def _empty_token_fp8(size): +def _empty_token_fp8(size, block_size=128): *dims, k = size return ( torch.empty(size, device="cuda", dtype=torch.float8_e4m3fn), torch.ones( - (*dims, ceil_div(k, _BLOCK_SIZE)), device="cuda", dtype=torch.float32 + (*dims, ceil_div(k, block_size)), device="cuda", dtype=torch.float32 ), ) -def _empty_block_fp8(size): +def _empty_block_fp8(size, block_size=128): *dims, n, k = size return ( torch.empty(size, device="cuda", dtype=torch.float8_e4m3fn), torch.ones( - (*dims, ceil_div(n, _BLOCK_SIZE), ceil_div(k, _BLOCK_SIZE)), + (*dims, ceil_div(n, block_size), ceil_div(k, block_size)), device="cuda", dtype=torch.float32, ), @@ -399,9 +412,12 @@ _BLOCK_SIZE = 128 class _NormalWarmupExecutor(_BaseWarmupExecutor): - def __init__(self, max_m: int, n: int, k: int, num_groups: int): - self.lhs_q, self.lhs_s = _empty_token_fp8((max_m, k)) - self.rhs_q, self.rhs_s = _empty_block_fp8((n, k)) + def __init__( + self, max_m: int, n: int, k: int, num_groups: int, block_size: int = 128 + ): + self.recipe = (1, block_size, block_size) + self.lhs_q, self.lhs_s = _empty_token_fp8((max_m, k), block_size) + self.rhs_q, self.rhs_s = _empty_block_fp8((n, k), block_size) self.out = torch.empty((max_m, n), device="cuda", dtype=torch.bfloat16) def execute(self, m): @@ -409,6 +425,7 @@ class _NormalWarmupExecutor(_BaseWarmupExecutor): (self.lhs_q[:m], self.lhs_s[:m]), (self.rhs_q, self.rhs_s), self.out[:m], + recipe=self.recipe, ) diff --git a/python/sglang/srt/layers/deep_gemm_wrapper/entrypoint.py b/python/sglang/srt/layers/deep_gemm_wrapper/entrypoint.py index 6134374bc3..7fff7d00bb 100644 --- a/python/sglang/srt/layers/deep_gemm_wrapper/entrypoint.py +++ b/python/sglang/srt/layers/deep_gemm_wrapper/entrypoint.py @@ -220,11 +220,16 @@ def gemm_nt_f8f8bf16( lhs: Tuple[torch.Tensor, torch.Tensor], rhs: Tuple[torch.Tensor, torch.Tensor], out: torch.Tensor, + recipe: Optional[Tuple[int, int, int]] = None, ): m, k = lhs[0].shape n, _ = rhs[0].shape num_groups = 1 - kernel_type = compile_utils.DeepGemmKernelType.GEMM_NT_F8F8BF16 + kernel_type = ( + compile_utils.DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32 + if recipe == (1, 32, 32) + else compile_utils.DeepGemmKernelType.GEMM_NT_F8F8BF16 + ) _sanity_check_input(lhs) _sanity_check_input(rhs) @@ -234,6 +239,7 @@ def gemm_nt_f8f8bf16( lhs, rhs, out, + **({"recipe": recipe} if recipe is not None else {}), ) diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index 6bb9b6867f..5542b41666 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -53,6 +53,7 @@ from sglang.srt.layers.quantization.base_config import ( QuantizeMethodBase, ) from sglang.srt.layers.quantization.fp8_utils import ( + _deepgemm_block_fp8_supported, _use_aiter_bpreshuffle_gfx95, apply_fp8_linear, block_fp8_scale_to_mxfp8_e8m0, @@ -729,6 +730,10 @@ class Fp8LinearMethod(LinearMethodBase): ) def process_weights_after_loading_block_quant(self, layer: Module) -> None: + use_deepgemm_runner = ( + getattr(self.w8a8_block_fp8_linear, "func", self.w8a8_block_fp8_linear) + is deepgemm_w8a8_block_fp8_linear_with_fallback + ) if self.convert_mxfp8_to_block: from sglang.srt.layers.quantization.mxfp8_block_convert import ( convert_mxfp8_weight_to_block_fp8, @@ -810,10 +815,6 @@ class Fp8LinearMethod(LinearMethodBase): return else: # Requantize block scales to UE8M0 when DeepGEMM is the active runner. - use_deepgemm_runner = ( - self.w8a8_block_fp8_linear - is deepgemm_w8a8_block_fp8_linear_with_fallback - ) requant_block_scale_ue8m0_for_deepgemm( layer.weight, layer.weight_scale_inv, @@ -833,6 +834,14 @@ class Fp8LinearMethod(LinearMethodBase): _is_cuda and get_platform().is_sm90 and envs.SGLANG_OPT_HOPPER_BLOCK_FP8_BF16.get() + and not ( + use_deepgemm_runner + and _deepgemm_block_fp8_supported( + weight.shape, + self.weight_block_size, + getattr(layer, "orig_dtype", None), + ) + ) and weight.is_cuda and weight.dtype == torch.float8_e4m3fn and self.weight_block_size == [32, 32] diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 227eaea3bb..3b560c587a 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -620,19 +620,32 @@ def dispatch_w8a8_block_fp8_linear( 1. The --fp8-gemm-backend server argument (preferred) 2. Auto-detection based on hardware capabilities """ - # Only Triton reads the block size at launch; DeepGEMM, the FlashInfer - # groupwise kernels and CUTLASS take 128-wide K blocks only. + backend = get_fp8_gemm_runner_backend() + if ( + weight_block_size == [32, 32] + and get_platform().is_sm90 + and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM + and (backend.is_auto() or backend.is_deep_gemm()) + ): + return partial( + deepgemm_w8a8_block_fp8_linear_with_fallback, + act_scale_ue8m0=act_scale_ue8m0, + ) + + # Other non-128 layouts still require Triton or the separate MXFP8 route. if weight_block_size is not None and weight_block_size[1] != 128: return partial(triton_w8a8_block_fp8_linear, act_scale_ue8m0=act_scale_ue8m0) - backend = get_fp8_gemm_runner_backend() - # Handle explicit backend selection via --fp8-gemm-backend if not backend.is_auto(): - return _dispatch_explicit_backend(backend) - - # Auto mode: Select based purely on hardware/backend availability - return _dispatch_auto_backend() + linear = _dispatch_explicit_backend(backend) + else: + # Auto mode: Select based purely on hardware/backend availability + linear = _dispatch_auto_backend() + # Preserve the unwrapped callable when no activation policy needs binding. + if act_scale_ue8m0 and linear is deepgemm_w8a8_block_fp8_linear_with_fallback: + return partial(linear, act_scale_ue8m0=True) + return linear def torch_w8a8_block_fp8_linear( @@ -1268,6 +1281,17 @@ def cutlass_w8a8_block_fp8_linear_with_fallback( return output.to(dtype=input_2d.dtype).view(*output_shape) +def _deepgemm_block_fp8_supported( + weight_shape: Tuple[int, int], block_size: List[int], output_dtype: torch.dtype +) -> bool: + if output_dtype != torch.bfloat16: + return False + n, k = weight_shape + if list(block_size) == [32, 32] and get_platform().is_sm90: + return n % 8 == 0 and k % 32 == 0 + return list(block_size) == [128, 128] and n % 64 == 0 and k % 128 == 0 + + def deepgemm_w8a8_block_fp8_linear_with_fallback( input: torch.Tensor, weight: torch.Tensor, @@ -1275,43 +1299,16 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback( weight_scale: torch.Tensor, input_scale: Optional[torch.Tensor] = None, bias: Optional[torch.Tensor] = None, + act_scale_ue8m0: bool = False, + weight_bf16: Optional[torch.Tensor] = None, ) -> torch.Tensor: if input_scale is not None: - # Pre-quantized activation (SGLANG_OPT_MOE_QUANT_ONCE): ``input`` is - # the fp8 per-token-group-128 q with rows padded to a multiple of 4 - # and ``input_scale`` the matching column-major fp32 scales - # (stride == (1, padded_rows)) -- identical to the MN-major - # TMA-aligned layout this path's own quant would produce below. - # Output keeps the padded row count; the caller slices back. - # UE8M0 packed scales (Blackwell DeepGEMM) use a different layout; - # the caller gates on it. assert not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 assert input.dtype == torch.float8_e4m3fn - assert weight.shape[0] % 64 == 0 and weight.shape[1] % 128 == 0, ( - "pre-quantized fp8 input requires DeepGEMM-supported weight shapes " - f"(got {tuple(weight.shape)})" - ) - input_2d = input.view(-1, input.shape[-1]) - output = w8a8_block_fp8_matmul_deepgemm( - input_2d, - weight, - input_scale, - weight_scale, - block_size, - output_dtype=torch.bfloat16, - ) - if bias is not None: - output += bias - return output.view(*input.shape[:-1], weight.shape[0]) - - output_dtype = input.dtype - dtype_supported = output_dtype == torch.bfloat16 + output_dtype = torch.bfloat16 if input_scale is not None else input.dtype # TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737 - shape_supported = weight.shape[0] % 64 == 0 and weight.shape[1] % 128 == 0 - block_supported = list(block_size) == [128, 128] - - if not (shape_supported and dtype_supported and block_supported): + if not _deepgemm_block_fp8_supported(weight.shape, block_size, output_dtype): # fall back to triton # If weight_scale is in UE8M0 packed format (int32), convert back to float32 # UE8M0 format has shape (N, K//block_k//4) with dtype int32 @@ -1321,13 +1318,42 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback( weight_scale, weight.shape, block_size ) return triton_w8a8_block_fp8_linear( - input, weight, block_size, weight_scale, input_scale, bias + input, + weight, + block_size, + weight_scale, + input_scale, + bias, + act_scale_ue8m0=act_scale_ue8m0, + weight_bf16=weight_bf16, ) + if input_scale is not None: + # Pre-quantized activations retain their per-token group size and + # fp32 scales. DeepGEMM aligns scale storage for TMA when needed. + input_2d = input.view(-1, input.shape[-1]) + output = w8a8_block_fp8_matmul_deepgemm( + input_2d, + weight, + input_scale, + weight_scale, + block_size, + output_dtype=output_dtype, + ) + if bias is not None: + output += bias + return output.view(*input.shape[:-1], weight.shape[0]) + input_2d = input.view(-1, input.shape[-1]) output_shape = [*input.shape[:-1], weight.shape[0]] - if not _is_musa: + if get_platform().is_sm90 and act_scale_ue8m0: + # Keep the checkpoint's power-of-two activation quantization. SM90 + # consumes fp32 scales; DeepGEMM aligns their storage for TMA. + q_input, x_scale = sglang_per_token_group_quant_fp8( + input_2d, block_size[1], scale_ue8m0=True + ) + elif not _is_musa: q_input, x_scale = sglang_per_token_group_quant_fp8( input_2d, block_size[1], diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 3df778ffaa..91c04733e7 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -1775,7 +1775,8 @@ class DeepseekV2MoE(nn.Module): if gate_up.weight.shape[0] % 64 != 0 or gate_up.weight.shape[1] % 128 != 0: return False, "gate_up weight shape unsupported by deepgemm" else: - return False, f"w8a8 linear backend {linear_fn.__name__} unsupported" + backend_name = getattr(linear_fn, "__name__", type(linear_fn).__name__) + return False, f"w8a8 linear backend {backend_name} unsupported" # Routed side: standard dispatcher + triton fused func with dynamic # per-token-group-128 fp8 activation quant. experts = self.experts diff --git a/test/registered/unit/layers/deep_gemm_wrapper/test_warmup_memory.py b/test/registered/unit/layers/deep_gemm_wrapper/test_warmup_memory.py index 480a6381b1..1fa07b9a9e 100644 --- a/test/registered/unit/layers/deep_gemm_wrapper/test_warmup_memory.py +++ b/test/registered/unit/layers/deep_gemm_wrapper/test_warmup_memory.py @@ -96,6 +96,11 @@ class TestWarmupMemory(CustomTestCase): (16384, 4096, 7168, 32), # V3 DP8/EP8 startup OOM shape. ): for kernel_type in DeepGemmKernelType: + if ( + kernel_type == DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32 + and architecture != "hopper" + ): + continue # Native block32 is only dispatched on Hopper. with self.subTest( architecture=architecture, kernel_type=kernel_type, max_m=max_m ): diff --git a/test/registered/unit/layers/quantization/test_fp8_blockwise_linear_backends.py b/test/registered/unit/layers/quantization/test_fp8_blockwise_linear_backends.py index 56317e09fa..536f3b5b37 100644 --- a/test/registered/unit/layers/quantization/test_fp8_blockwise_linear_backends.py +++ b/test/registered/unit/layers/quantization/test_fp8_blockwise_linear_backends.py @@ -192,6 +192,30 @@ class TestFp8BlockwiseLinearBackends(_LinearBackendCheck): def test_deep_gemm(self): self._run("deep_gemm") + @unittest.skipUnless( + get_device_sm() == 90, "SM90 UE8M0 activation-scale regression" + ) + def test_deep_gemm_ue8m0_activation_scales(self): + torch.manual_seed(7) + x = torch.randn(3, 128, device="cuda", dtype=torch.bfloat16) + weight = torch.eye(128, device="cuda").to(torch.float8_e4m3fn) + weight_scale = torch.ones(1, 1, device="cuda") + q, scale = fp8_utils.sglang_per_token_group_quant_fp8(x, 128, scale_ue8m0=True) + ref = (q.float() * scale).to(torch.bfloat16) + for backend in (Fp8GemmRunnerBackend.AUTO, Fp8GemmRunnerBackend.DEEP_GEMM): + with mock.patch.object(fp8_utils, "FP8_GEMM_RUNNER_BACKEND", backend): + self.assertIs( + fp8_utils.dispatch_w8a8_block_fp8_linear([128, 128]), + fp8_utils.deepgemm_w8a8_block_fp8_linear_with_fallback, + ) + linear = fp8_utils.dispatch_w8a8_block_fp8_linear( + [128, 128], act_scale_ue8m0=True + ) + for n in (128, 96): + with self.subTest(backend=backend, n=n): + out = linear(x, weight[:n], [128, 128], weight_scale) + torch.testing.assert_close(out, ref[:, :n], rtol=0, atol=0) + def test_flashinfer_trtllm(self): self._run("flashinfer_trtllm") @@ -339,6 +363,82 @@ class TestBlockFp8AsMxfp8Linear(_LinearBackendCheck): plain_layer.quant_method.apply(plain_layer, Mxfp8SwizzledInput(q, s)) +@unittest.skipUnless(get_device_sm() == 90, "Native block32 DeepGEMM requires SM90") +class TestSm90Block32Linear(_LinearBackendCheck): + _build_layer = staticmethod(_build_block32_layer) + _shapes = [(1, 96, 288), (6, 5120, 288), (65, 1792, 5120), (512, 512, 512)] + + def test_deepgemm_and_auto_use_native_blocks(self): + for backend in ("deep_gemm", "auto"): + with ( + self.subTest(backend=backend), + mock.patch.object( + fp8_utils, + "triton_w8a8_block_fp8_linear", + side_effect=AssertionError( + "native block32 unexpectedly fell back to Triton" + ), + ), + ): + self._check_backend(backend, [backend], self._shapes, self._build_layer) + + def test_triton_override(self): + self._check_backend("triton", ["triton"], self._shapes, self._build_layer) + + def test_unsupported_shape_keeps_triton_and_bf16_fallback(self): + torch.manual_seed(7) + with ( + mock.patch.object( + fp8_utils, "FP8_GEMM_RUNNER_BACKEND", Fp8GemmRunnerBackend.DEEP_GEMM + ), + mock.patch.object( + fp8_utils.envs.SGLANG_OPT_HOPPER_BLOCK_FP8_BF16, + "get", + return_value=True, + ), + mock.patch.object( + fp8_utils, + "w8a8_block_fp8_matmul_deepgemm", + side_effect=AssertionError("unsupported shape reached DeepGEMM"), + ), + ): + layer, w_dequant = self._build_layer(32, 64) + layer.weight.data = layer.weight.data[:10].contiguous() + layer.quant_method.process_weights_after_loading(layer) + self.assertIsNotNone(getattr(layer, "_block_fp8_bf16_weight", None)) + x = torch.randn(65, 64, device="cuda", dtype=torch.bfloat16) / 10 + q, scales = fp8_utils.sglang_per_token_group_quant_fp8( + x, 32, scale_ue8m0=True + ) + ref = fp8_utils.dequant_group_fp8_to_bf16(q, scales).float() + ref = ref @ w_dequant[:10].T + for inputs in (x, (q, scales)): + out = layer.quant_method.apply(layer, inputs) + assert_output_close(self, out, ref, rtol=5e-2, atol=1e-1) + + def test_preserves_checkpoint_and_prequantized_input(self): + torch.manual_seed(7) + with mock.patch.object( + fp8_utils, "FP8_GEMM_RUNNER_BACKEND", Fp8GemmRunnerBackend.DEEP_GEMM + ): + layer, _ = self._build_layer(96, 288) + original_weight = layer.weight.detach().clone() + layer.quant_method.process_weights_after_loading(layer) + self.assertIsNone(getattr(layer, "_block_fp8_bf16_weight", None)) + torch.testing.assert_close( + layer.weight.float(), original_weight.float(), rtol=0, atol=0 + ) + self.assertEqual(tuple(layer.weight_scale_inv.shape), (3, 9)) + x = torch.randn(5, 288, device="cuda", dtype=torch.bfloat16) / 10 + q, scales = fp8_utils.sglang_per_token_group_quant_fp8( + x, 32, scale_ue8m0=True + ) + bias = torch.randn(96, device="cuda", dtype=torch.bfloat16) + plain = layer.quant_method.apply(layer, x, bias) + quantized = layer.quant_method.apply(layer, (q, scales), bias) + torch.testing.assert_close(plain, quantized, rtol=0, atol=0) + + @unittest.skipUnless( "flashinfer_cutedsl" in _block32_backends(), "block-fp8-as-MXFP8 prefill tuning needs the FlashInfer CuTe-DSL kernel",