[AMD] Fix Load and Inference of MLA models with Quark PTPC FP8 attention on ROCm (#28734)
This commit is contained in:
@@ -601,6 +601,16 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
enable_dual_stream: bool,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
# fused_rms_fp8_group_quant with output_unquantized_inp1=True produces a
|
||||
# (fp8, scale, bf16) 3-tuple; extract the bf16 for the unquantized wk layer.
|
||||
if isinstance(x, tuple):
|
||||
if len(x) >= 3:
|
||||
x = x[2]
|
||||
else:
|
||||
raise ValueError(
|
||||
f"_get_q_k_bf16 received a {len(x)}-tuple; expected a plain tensor "
|
||||
"or a 3-tuple (fp8, scale, bf16) from fused_rms_fp8_group_quant."
|
||||
)
|
||||
weights_raw = None
|
||||
if enable_dual_stream:
|
||||
current_stream = torch.cuda.current_stream()
|
||||
@@ -688,6 +698,16 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
*,
|
||||
apply_hadamard: bool = True,
|
||||
):
|
||||
# fused_rms_fp8_group_quant with output_unquantized_inp1=True produces a
|
||||
# (fp8, scale, bf16) 3-tuple; extract the bf16 for the unquantized wk layer.
|
||||
if isinstance(x, tuple):
|
||||
if len(x) >= 3:
|
||||
x = x[2]
|
||||
else:
|
||||
raise ValueError(
|
||||
f"_get_k_bf16 received a {len(x)}-tuple; expected a plain tensor "
|
||||
"or a 3-tuple (fp8, scale, bf16) from fused_rms_fp8_group_quant."
|
||||
)
|
||||
# Non-fusion path only; self.wk does not exist when fusion is on.
|
||||
key, _ = self.wk(x)
|
||||
key = self.k_norm(key)
|
||||
|
||||
@@ -1990,6 +1990,9 @@ def channel_quant_to_tensor_quant(
|
||||
x_q_channel: torch.Tensor,
|
||||
x_s: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
# Reshape per-output-channel scale [N] -> [N, 1, ...] to broadcast over K dims.
|
||||
while x_s.dim() < x_q_channel.dim():
|
||||
x_s = x_s.unsqueeze(-1)
|
||||
x_dq_channel = x_q_channel.to(torch.float32) * x_s
|
||||
x_q_tensor, scale = (
|
||||
scaled_fp8_quant(x_dq_channel)
|
||||
@@ -2355,9 +2358,10 @@ def apply_fp8_ptpc_linear(
|
||||
compressed_tensor_quant: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""FP8 per-token per-channel linear. Only used with the aiter (ROCm) backend."""
|
||||
# Handle pre-quantized (fp8_tensor, scale) tuple from fused RMSNorm+Quant
|
||||
# Handle pre-quantized (fp8_tensor, scale[, bf16]) tuple from fused RMSNorm+Quant.
|
||||
# The optional 3rd element is the unquantized bf16 tensor kept for DSA; ignore it.
|
||||
if isinstance(input, tuple):
|
||||
q_input, x_scale = input
|
||||
q_input, x_scale = input[0], input[1]
|
||||
q_input = q_input.view(-1, q_input.shape[-1])
|
||||
output_shape = [*q_input.shape[:-1], weight.shape[0]]
|
||||
output = aiter.gemm_a8w8_bpreshuffle(
|
||||
|
||||
@@ -174,7 +174,14 @@ class QuarkW8A8Fp8(QuarkLinearScheme):
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
|
||||
# Activations must be a plain bf16 tensor: per-channel FP8 (weight_scale
|
||||
# [N, 1]) is gated off the aiter fused RMSNorm+quant kernel upstream by
|
||||
# _is_block_scale_fp8, so the per-token quant happens in apply_fp8_linear.
|
||||
assert not isinstance(x, tuple), (
|
||||
"quark W8A8 FP8 linear received a pre-quantized tuple; a fused "
|
||||
"RMSNorm+quant producer was not gated off by _is_block_scale_fp8 "
|
||||
"(per-channel FP8 must fall through to the plain bf16 path)."
|
||||
)
|
||||
return apply_fp8_linear(
|
||||
x,
|
||||
layer.weight,
|
||||
|
||||
Reference in New Issue
Block a user