[AMD] Fix Load and Inference of MLA models with Quark PTPC FP8 attention on ROCm (#28734)

This commit is contained in:
Colin Z
2026-09-29 11:20:10 -07:00
committed by GitHub
parent affc602f51
commit 875dd41e6f
4 changed files with 99 additions and 3 deletions
@@ -601,6 +601,16 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
enable_dual_stream: bool,
forward_batch: ForwardBatch,
):
# fused_rms_fp8_group_quant with output_unquantized_inp1=True produces a
# (fp8, scale, bf16) 3-tuple; extract the bf16 for the unquantized wk layer.
if isinstance(x, tuple):
if len(x) >= 3:
x = x[2]
else:
raise ValueError(
f"_get_q_k_bf16 received a {len(x)}-tuple; expected a plain tensor "
"or a 3-tuple (fp8, scale, bf16) from fused_rms_fp8_group_quant."
)
weights_raw = None
if enable_dual_stream:
current_stream = torch.cuda.current_stream()
@@ -688,6 +698,16 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
*,
apply_hadamard: bool = True,
):
# fused_rms_fp8_group_quant with output_unquantized_inp1=True produces a
# (fp8, scale, bf16) 3-tuple; extract the bf16 for the unquantized wk layer.
if isinstance(x, tuple):
if len(x) >= 3:
x = x[2]
else:
raise ValueError(
f"_get_k_bf16 received a {len(x)}-tuple; expected a plain tensor "
"or a 3-tuple (fp8, scale, bf16) from fused_rms_fp8_group_quant."
)
# Non-fusion path only; self.wk does not exist when fusion is on.
key, _ = self.wk(x)
key = self.k_norm(key)
@@ -1990,6 +1990,9 @@ def channel_quant_to_tensor_quant(
x_q_channel: torch.Tensor,
x_s: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
# Reshape per-output-channel scale [N] -> [N, 1, ...] to broadcast over K dims.
while x_s.dim() < x_q_channel.dim():
x_s = x_s.unsqueeze(-1)
x_dq_channel = x_q_channel.to(torch.float32) * x_s
x_q_tensor, scale = (
scaled_fp8_quant(x_dq_channel)
@@ -2355,9 +2358,10 @@ def apply_fp8_ptpc_linear(
compressed_tensor_quant: bool = False,
) -> torch.Tensor:
"""FP8 per-token per-channel linear. Only used with the aiter (ROCm) backend."""
# Handle pre-quantized (fp8_tensor, scale) tuple from fused RMSNorm+Quant
# Handle pre-quantized (fp8_tensor, scale[, bf16]) tuple from fused RMSNorm+Quant.
# The optional 3rd element is the unquantized bf16 tensor kept for DSA; ignore it.
if isinstance(input, tuple):
q_input, x_scale = input
q_input, x_scale = input[0], input[1]
q_input = q_input.view(-1, q_input.shape[-1])
output_shape = [*q_input.shape[:-1], weight.shape[0]]
output = aiter.gemm_a8w8_bpreshuffle(
@@ -174,7 +174,14 @@ class QuarkW8A8Fp8(QuarkLinearScheme):
x: torch.Tensor,
bias: Optional[torch.Tensor] = None,
) -> torch.Tensor:
# Activations must be a plain bf16 tensor: per-channel FP8 (weight_scale
# [N, 1]) is gated off the aiter fused RMSNorm+quant kernel upstream by
# _is_block_scale_fp8, so the per-token quant happens in apply_fp8_linear.
assert not isinstance(x, tuple), (
"quark W8A8 FP8 linear received a pre-quantized tuple; a fused "
"RMSNorm+quant producer was not gated off by _is_block_scale_fp8 "
"(per-channel FP8 must fall through to the plain bf16 path)."
)
return apply_fp8_linear(
x,
layer.weight,