diff --git a/python/sglang/kernels/ops/activation/silu_and_mul_clamp_hip.py b/python/sglang/kernels/ops/activation/silu_and_mul_clamp_hip.py index c840233dc6..a4c7a3b0bc 100644 --- a/python/sglang/kernels/ops/activation/silu_and_mul_clamp_hip.py +++ b/python/sglang/kernels/ops/activation/silu_and_mul_clamp_hip.py @@ -68,9 +68,12 @@ def _silu_and_mul_clamp_kernel( ) -def silu_and_mul_clamp_fp8_grid_supported(intermediate: int) -> bool: - """The fp8-grid epilogue needs whole 32-wide groups inside one program.""" - return intermediate % 32 == 0 and intermediate <= 1024 +def silu_and_mul_clamp_fp8_grid_supported( + intermediate: int, multi_block: bool = False +) -> bool: + """The fp8-grid epilogue needs whole 32-wide groups inside one program; 1024-wide programs + keep that for any multiple of 32 (multi_block).""" + return intermediate % 32 == 0 and (multi_block or intermediate <= 1024) def silu_and_mul_clamp_triton( @@ -87,8 +90,8 @@ def silu_and_mul_clamp_triton( inter_size = N // 2 fp8_grid = fp8_grid or emit_fp8 if fp8_grid: - assert silu_and_mul_clamp_fp8_grid_supported(inter_size), ( - f"inter_size {inter_size}: the fp8-grid epilogue needs a multiple of 32 up to 1024" + assert silu_and_mul_clamp_fp8_grid_supported(inter_size, multi_block=True), ( + f"inter_size {inter_size}: the fp8-grid epilogue needs a multiple of 32" ) out = torch.empty( (M, inter_size), diff --git a/python/sglang/kernels/ops/gemm/gfx95_batched_gemm_bf16_fp8_grid.py b/python/sglang/kernels/ops/gemm/gfx95_batched_gemm_bf16_fp8_grid.py index 0df1e37fef..5ad9566846 100644 --- a/python/sglang/kernels/ops/gemm/gfx95_batched_gemm_bf16_fp8_grid.py +++ b/python/sglang/kernels/ops/gemm/gfx95_batched_gemm_bf16_fp8_grid.py @@ -11,7 +11,10 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import fp8_grid_round +from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( + fp8_grid_quant, + fp8_grid_round, +) # 16 x 32 x 512, 2 warps: the fastest N tile holding whole 32-groups; nonkdim 16 is aiter's MFMA _BLOCK_M, _BLOCK_N, _BLOCK_K = 16, 32, 512 @@ -50,11 +53,15 @@ def _batched_gemm_bf16_fp8_grid_kernel( stride_cb, stride_cm, stride_cn, + scale_ptr, + stride_sb, + stride_sm, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, EVEN_K: tl.constexpr, FP8_GRID: tl.constexpr, + EMIT_FP8: tl.constexpr, cache_modifier: tl.constexpr, num_warps: tl.constexpr, num_stages: tl.constexpr, @@ -117,12 +124,21 @@ def _batched_gemm_bf16_fp8_grid_kernel( a_ptrs += BLOCK_SIZE_K * stride_ak b_ptrs += BLOCK_SIZE_K * stride_bk - c = accumulator.to(c_ptr.type.element_ty) - if FP8_GRID: - # the consumer's fake-quant on the bf16-rounded output: one ue8m0 group per 32 N elements - xg = tl.reshape(c.to(tl.float32), (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32)) - c = tl.reshape(fp8_grid_round(xg), (BLOCK_SIZE_M, BLOCK_SIZE_N)) - c = c.to(c_ptr.type.element_ty) + if EMIT_FP8: + # the consumer's own quant of the bf16-rounded output: fp8 codes + one ue8m0 per 32 N + xg = tl.reshape( + accumulator.to(tl.bfloat16).to(tl.float32), + (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32), + ) + q8, e8 = fp8_grid_quant(xg) + c = tl.reshape(q8, (BLOCK_SIZE_M, BLOCK_SIZE_N)) + else: + c = accumulator.to(c_ptr.type.element_ty) + if FP8_GRID: + # the consumer's fake-quant on the bf16-rounded output: one ue8m0 group per 32 N elements + xg = tl.reshape(c.to(tl.float32), (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32)) + c = tl.reshape(fp8_grid_round(xg), (BLOCK_SIZE_M, BLOCK_SIZE_N)) + c = c.to(c_ptr.type.element_ty) offs_cm = tl.cast(pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M), tl.int64) offs_cn = tl.cast(pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N), tl.int64) @@ -134,6 +150,16 @@ def _batched_gemm_bf16_fp8_grid_kernel( ) c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) tl.store(c_ptrs, c, mask=c_mask) + if EMIT_FP8: + offs_sn = pid_n * (BLOCK_SIZE_N // 32) + tl.arange(0, BLOCK_SIZE_N // 32) + tl.store( + scale_ptr + + stride_sb * batch_id + + stride_sm * offs_cm[:, None] + + offs_sn[None, :], + tl.reshape(e8, (BLOCK_SIZE_M, BLOCK_SIZE_N // 32)).to(tl.uint8), + mask=(offs_cm[:, None] < M) & (offs_sn[None, :] < N // 32), + ) @triton.jit @@ -208,10 +234,14 @@ def _batched_gemm_split_k_reduce_kernel( N, stride_cb, stride_cm, + scale_ptr, + stride_sb, + stride_sm, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, SPLIT_K: tl.constexpr, FP8_GRID: tl.constexpr, + EMIT_FP8: tl.constexpr, ): """Grid (G, row tiles x N tiles): sums the partials of one output tile in split order, rounds to bf16 and, with FP8_GRID, onto the consumer's fp8 grid.""" @@ -231,11 +261,19 @@ def _batched_gemm_split_k_reduce_kernel( + (tile + tl.arange(0, BLOCK_SIZE_M))[:, None] * N + offs_n[None, :] ) - c = acc.to(c_ptr.type.element_ty) - if FP8_GRID: - xg = tl.reshape(c.to(tl.float32), (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32)) - c = tl.reshape(fp8_grid_round(xg), (BLOCK_SIZE_M, BLOCK_SIZE_N)) - c = c.to(c_ptr.type.element_ty) + if EMIT_FP8: + xg = tl.reshape( + acc.to(tl.bfloat16).to(tl.float32), + (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32), + ) + q8, e8 = fp8_grid_quant(xg) + c = tl.reshape(q8, (BLOCK_SIZE_M, BLOCK_SIZE_N)) + else: + c = acc.to(c_ptr.type.element_ty) + if FP8_GRID: + xg = tl.reshape(c.to(tl.float32), (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32)) + c = tl.reshape(fp8_grid_round(xg), (BLOCK_SIZE_M, BLOCK_SIZE_N)) + c = c.to(c_ptr.type.element_ty) c_ptrs = ( c_ptr + stride_cb * batch_id @@ -243,6 +281,16 @@ def _batched_gemm_split_k_reduce_kernel( + offs_n[None, :] ) tl.store(c_ptrs, c, mask=(offs_m[:, None] < M) & (offs_n[None, :] < N)) + if EMIT_FP8: + offs_sn = pid_n * (BLOCK_SIZE_N // 32) + tl.arange(0, BLOCK_SIZE_N // 32) + tl.store( + scale_ptr + + stride_sb * batch_id + + stride_sm * tl.cast(offs_m, tl.int64)[:, None] + + offs_sn[None, :], + tl.reshape(e8, (BLOCK_SIZE_M, BLOCK_SIZE_N // 32)).to(tl.uint8), + mask=(offs_m[:, None] < M) & (offs_sn[None, :] < N // 32), + ) def _split_k_applies(T: int, D: int, R: int) -> bool: @@ -255,7 +303,11 @@ def _split_k_applies(T: int, D: int, R: int) -> bool: def _batched_gemm_split_k( - x: torch.Tensor, w: torch.Tensor, out: torch.Tensor, fp8_grid: bool + x: torch.Tensor, + w: torch.Tensor, + out: torch.Tensor, + fp8_grid: bool, + scale: Optional[torch.Tensor] = None, ) -> None: T, G, D = x.shape R = w.shape[1] @@ -293,10 +345,14 @@ def _batched_gemm_split_k( R, R, G * R, + scale if scale is not None else out, + R // 32, + G * R // 32, BLOCK_SIZE_M=_BLOCK_M, BLOCK_SIZE_N=_BLOCK_N, SPLIT_K=_SPLIT_K, FP8_GRID=fp8_grid, + EMIT_FP8=scale is not None, num_warps=_NUM_WARPS, ) @@ -306,26 +362,38 @@ def batched_gemm_bf16_fp8_grid( w: torch.Tensor, fp8_grid: bool = True, split_k: Optional[bool] = None, -) -> torch.Tensor: + emit_fp8: bool = False, +): """x [T, G, D] bf16, w [G, R, D] bf16 -> [T, G * R] bf16 with out[t, g*R:(g+1)*R] = x[t, g] @ w[g]^T, on the fp8 grid when fp8_grid; split_k=None picks the regime by shape, - False keeps the single launch at every T (deterministic inference).""" + False keeps the single launch at every T (deterministic inference). + emit_fp8 returns (fp8 e4m3 [T, G * R], ue8m0 uint8 [T, G * R // 32]) instead.""" assert x.dim() == 3 and w.dim() == 3, (x.shape, w.shape) T, G, D = x.shape assert w.shape[0] == G and w.shape[2] == D, (x.shape, w.shape) R = w.shape[1] assert x.dtype == torch.bfloat16 and w.dtype == torch.bfloat16 assert x.stride(2) == 1 and w.is_contiguous() - assert not fp8_grid or R % 32 == 0, R - out = torch.empty((T, G * R), dtype=torch.bfloat16, device=x.device) + assert not (fp8_grid or emit_fp8) or R % 32 == 0, R + out = torch.empty( + (T, G * R), + dtype=torch.float8_e4m3fn if emit_fp8 else torch.bfloat16, + device=x.device, + ) + scale = ( + torch.empty((T, G * R // 32), dtype=torch.uint8, device=x.device) + if emit_fp8 + else None + ) + result = (out, scale) if emit_fp8 else out if T == 0: - return out + return result if split_k is None: split_k = _split_k_applies(T, D, R) if split_k: assert _split_k_applies(T, D, R), (T, D, R) - _batched_gemm_split_k(x, w, out, fp8_grid) - return out + _batched_gemm_split_k(x, w, out, fp8_grid, scale) + return result block_m, block_n, block_k, num_warps = _tile_config(T) grid = (G, triton.cdiv(T, block_m) * triton.cdiv(R, block_n)) _batched_gemm_bf16_fp8_grid_kernel[grid]( @@ -344,15 +412,19 @@ def batched_gemm_bf16_fp8_grid( R, G * R, 1, + scale if scale is not None else out, + R // 32, + G * R // 32, BLOCK_SIZE_M=block_m, BLOCK_SIZE_N=block_n, BLOCK_SIZE_K=block_k, EVEN_K=(D % block_k == 0), FP8_GRID=fp8_grid, + EMIT_FP8=emit_fp8, cache_modifier=_CACHE_MODIFIER, num_warps=num_warps, num_stages=_NUM_STAGES, waves_per_eu=_WAVES_PER_EU, matrix_instr_nonkdim=_MFMA_NONKDIM, ) - return out + return result diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 483771531c..7d09409e6e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1592,6 +1592,12 @@ class Envs: # DSpark draft block on the HIP radix backend: build the attention metadata inside the # draft CUDA graph from the raw inputs instead of eagerly before every replay. SGLANG_HIP_DSPARK_DRAFT_RAW_METADATA = EnvBool(_default_hip) + # gfx950 MXFP8 dense routes (aiter group32 / native): the producer emits fp8 + ue8m0 for its + # consumer instead of bf16 plus a separate quant launch -- the shared expert's SwiGLU for + # down_proj, the wo_a GEMM for wo_b, and the FFN norm for the shared expert's gate_up. + SGLANG_HIP_SHARED_ACT_MXFP8 = EnvBool(_default_hip) + SGLANG_HIP_WO_A_MXFP8 = EnvBool(_default_hip) + SGLANG_HIP_FFN_NORM_MXFP8 = EnvBool(_default_hip) # cache, GEMM, and distributed SGLANG_OPT_FP8_WO_A_GEMM = EnvBool(True) diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_act.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_act.py index 9b62db3f8f..351228ead9 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_act.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_act.py @@ -10,6 +10,7 @@ from sglang.kernels.ops.activation.silu_and_mul_clamp_hip import ( silu_and_mul_clamp_fp8_grid_supported, silu_and_mul_clamp_triton, ) +from sglang.srt.environ import envs from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod @@ -33,7 +34,9 @@ def resolve_fused_clamp_route(mlp, half_width: int) -> None: quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native() or quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_aiter() ) - and silu_and_mul_clamp_fp8_grid_supported(half_width) + and silu_and_mul_clamp_fp8_grid_supported( + half_width, multi_block=envs.SGLANG_HIP_SHARED_ACT_MXFP8.get() + ) ) # the native and aiter MXFP8 routes take fp8 + ue8m0 straight from the epilogue mlp._hip_act_native_consumer = bool( diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_gfx95_dense.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_gfx95_dense.py index 360c10afa1..20680e5830 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_gfx95_dense.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_gfx95_dense.py @@ -165,12 +165,36 @@ def post_attention_norm(layer, x: torch.Tensor, coefficients=None) -> torch.Tens if coefficients is not None: coefficients.materialize() return norm(x) + if _ffn_norm_emits_mxfp8(layer, x.shape[0]): + x_quant, out = rmsnorm_with_sinkhorn( + x, norm.weight.data, norm.variance_epsilon, coefficients, emit_fp8=True + ) + # the shared expert's gate_up takes it in place of its own quant (_forward_shared_experts) + out._hip_mxfp8_operand = x_quant + return out _, out = rmsnorm_with_sinkhorn( x, norm.weight.data, norm.variance_epsilon, coefficients, fake_quant=False ) return out +_FFN_NORM_EMIT_MXFP8 = envs.SGLANG_HIP_FFN_NORM_MXFP8.get() + + +def _ffn_norm_emits_mxfp8(layer, num_tokens: int) -> bool: + """SGLANG_HIP_FFN_NORM_MXFP8: whether the FFN norm launch also emits fp8 + ue8m0 for the + shared expert's gate_up (its route consumes them at this token count).""" + if not _FFN_NORM_EMIT_MXFP8: + return False + consumer = getattr(layer, "_ffn_shared_mxfp8_consumer", None) + if consumer is None: + shared = getattr(getattr(layer, "mlp", None), "shared_experts", None) + consumer = layer._ffn_shared_mxfp8_consumer = _mxfp8_consumer( + getattr(shared, "gate_up_proj", None) + ) + return consumer + + def live_rows(activation, num_tokens: int): """The first num_tokens rows of a break input; the gfx950 fused q_norm hands the indexer its Fp8GridActivation wrapper, whose rows live in .x.""" @@ -196,15 +220,37 @@ def wo_b_takes_fp8_grid(attn) -> bool: return attn._wo_b_fp8_grid_operand -def wo_a_fp8_grid_matmul(o: torch.Tensor, wo_a: torch.Tensor, fp8_grid: bool): +_WO_A_EMIT_MXFP8 = envs.SGLANG_HIP_WO_A_MXFP8.get() + + +def wo_b_emits_mxfp8(attn, num_tokens: int) -> bool: + """SGLANG_HIP_WO_A_MXFP8: whether the wo_a GEMM hands wo_b fp8 + ue8m0 from its epilogue + (wo_b's route consumes them at this token count) instead of bf16 plus a separate quant.""" + if not _WO_A_EMIT_MXFP8 or _wo_a_fp8_grid_gemm is None: + return False + consumer = getattr(attn, "_wo_b_mxfp8_consumer", None) + if consumer is None: + consumer = attn._wo_b_mxfp8_consumer = _mxfp8_consumer(attn.wo_b) + return consumer + + +def wo_a_fp8_grid_matmul( + o: torch.Tensor, wo_a: torch.Tensor, fp8_grid: bool, emit_fp8: bool = False +): """o [T, G, D] @ wo_a [G, R, D]^T through the gfx950 fork of the aiter batched GEMM: bf16 [T, G, R], or with fp8_grid rounded onto wo_b's fp8 grid as - an Fp8GridActivation [T, G * R]; None when the fork did not import.""" + an Fp8GridActivation [T, G * R], or with emit_fp8 an Mxfp8Activation [T, G * R]; + None when the fork did not import.""" if _wo_a_fp8_grid_gemm is None: return None # the split-K regime ends at 64 rows, so a request's verify and decode rows would # take different reduction orders; deterministic inference keeps the single chain split_k = False if get_exec().deterministic.enable_deterministic_inference else None + if emit_fp8: + q, scale = _wo_a_fp8_grid_gemm( + o, wo_a, fp8_grid=False, split_k=split_k, emit_fp8=True + ) + return Mxfp8Activation(q, scale) y = _wo_a_fp8_grid_gemm(o, wo_a, fp8_grid=fp8_grid, split_k=split_k) if fp8_grid: return Fp8GridActivation(y) diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py index cc96490ab4..864f0f9c00 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py @@ -28,6 +28,7 @@ from sglang.srt.utils import is_gfx95_supported live_rows = gfx95_dense.live_rows wo_a_fp8_grid_matmul = gfx95_dense.wo_a_fp8_grid_matmul +wo_b_emits_mxfp8 = gfx95_dense.wo_b_emits_mxfp8 _is_gfx95_supported = is_gfx95_supported() diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 9d4d4d5640..b83c315ed6 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -259,6 +259,7 @@ _moe_quant_once_logged = False _enable_pcg_dsv2_dual_stream = ( _is_cuda and envs.SGLANG_ENABLE_PCG_DSV2_DUAL_STREAM.get() ) +_hip_shared_act_mxfp8 = _is_hip and envs.SGLANG_HIP_SHARED_ACT_MXFP8.get() class DeepseekV2MLP(nn.Module): @@ -411,7 +412,16 @@ class DeepseekV2MLP(nn.Module): if self.use_fused_clamp_act_mul and not self._fused_clamp_fp8_checked: _hip_act.resolve_fused_clamp_route(self, gate_up.shape[-1] // 2) - if self.use_fused_clamp_act_mul and self.swiglu_limit is not None: + # SGLANG_HIP_SHARED_ACT_MXFP8: an MXFP8 down_proj takes fp8 + ue8m0 straight from the Triton + # epilogue instead of aiter's bf16 output plus a separate quant launch. + if ( + self.use_fused_clamp_act_mul + and self.swiglu_limit is not None + and _hip_shared_act_mxfp8 + and self._hip_act_fp8_grid + ): + x = _hip_act.silu_and_mul_clamp(self, gate_up) + elif self.use_fused_clamp_act_mul and self.swiglu_limit is not None: from aiter.ops.triton.fusions.fused_clamp_act_mul import ( fused_clamp_act_mul, ) @@ -1733,6 +1743,9 @@ class DeepseekV2MoE(nn.Module): pre_quant_input: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, ): if (hidden_states.shape[0] > 0) and (self.num_fused_shared_experts == 0): + if pre_quant_input is None and _is_hip: + # SGLANG_HIP_FFN_NORM_MXFP8: the FFN norm launch's fp8 + ue8m0 of these rows + pre_quant_input = getattr(hidden_states, "_hip_mxfp8_operand", None) if pre_quant_input is not None: # SGLANG_OPT_MOE_QUANT_ONCE: (q, s) rows may be padded to a # multiple of 4; the padded rows flow through the MLP (all ops diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 53461b33ec..d3fa6ff49b 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -462,7 +462,8 @@ def _apply_wo_a_bf16_matmul( is_prefill: bool = False, fast_path: bool = False, fp8_grid: bool = False, -) -> torch.Tensor | Mxfp8SwizzledInput: + emit_fp8: bool = False, +) -> torch.Tensor | Mxfp8SwizzledInput | Fp8GridActivation | Mxfp8Activation: # o [T, G, D] @ wo_a [G, R, D] -> [T, G, R]; the fast paths below are gated # on the exact validated TP4 shapes and write token-major output directly. global _wo_a_aiter_batched_gemm_disabled @@ -528,7 +529,7 @@ def _apply_wo_a_bf16_matmul( return result # the 16-row decode kernels also serve V4.1's target-verify rows if (is_decode or is_target_verify) and hip_fast_path: - y = _hip.wo_a_fp8_grid_matmul(o, wo_a, fp8_grid) + y = _hip.wo_a_fp8_grid_matmul(o, wo_a, fp8_grid, emit_fp8=emit_fp8) if y is not None: return y if ( @@ -669,6 +670,10 @@ def _apply_gguf_grouped_wo_a( if TYPE_CHECKING: + from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( + Fp8GridActivation, + Mxfp8Activation, + ) from sglang.srt.layers.attention.deepseek_v4_backend import ( DeepseekV4AttnBackend, LateLayerTail, @@ -2579,6 +2584,8 @@ class MQALayer(MqaAttentionBase): fast_path=self.is_dsv41, fuse_mxfp8_quant=fuse_mxfp8_quant, fp8_grid=_is_hip and _hip.wo_a_emits_fp8_grid(self), + emit_fp8=_is_hip + and _hip.wo_b_emits_mxfp8(self, o.shape[0]), ) else: o = _apply_gguf_grouped_wo_a( diff --git a/test/registered/unit/models/test_deepseek_v4_unified_fp8_q_pair.py b/test/registered/unit/models/test_deepseek_v4_unified_fp8_q_pair.py index a1a4b17b5e..cf722f59e3 100644 --- a/test/registered/unit/models/test_deepseek_v4_unified_fp8_q_pair.py +++ b/test/registered/unit/models/test_deepseek_v4_unified_fp8_q_pair.py @@ -120,7 +120,8 @@ _NO_HIP_GLUE = SimpleNamespace( skip_head_pad=lambda attn: False, attention_inv_rope=lambda *args, **kwargs: None, wo_a_emits_fp8_grid=lambda attn: False, - wo_a_fp8_grid_matmul=lambda o, wo_a, fp8_grid: None, + wo_b_emits_mxfp8=lambda attn, num_tokens: False, + wo_a_fp8_grid_matmul=lambda o, wo_a, fp8_grid, emit_fp8=False: None, )