[AMD] Eliminate remaining gfx95 FP8 scale relayout copies (#41030)

This commit is contained in:
xiaofei-zheng
2026-10-09 21:27:38 +08:00
committed by GitHub
parent 78996e3f02
commit d5c3ae9752
5 changed files with 110 additions and 13 deletions
@@ -33,7 +33,9 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
from sglang.srt.layers.quantization.fp8_utils import (
_use_aiter_bpreshuffle_gfx95,
emit_transposed_bpreshuffle_scale,
materialize_bpreshuffle_fp8_scale_tuple,
view_aiter_fused_rms_transposed_fp8_scale_tuple,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.runtime_context import get_exec, get_parallel, get_platform
@@ -213,6 +215,10 @@ def _update_and_read_residual_aiter_fp8_group(
unquantized bf16 output rides along as a third element, so the DSA indexer
can skip dequantizing. post_residual_addition is not applied on this path."""
needs_bf16 = get_attn_tp_context().is_dsa
emit_transposed_scale = emit_transposed_bpreshuffle_scale(
hidden_states.shape[0],
on_bpreshuffle_gfx95=_use_aiter_bpreshuffle_gfx95,
)
output, unquantized, _, residual_out = fused_rms_fp8_group_quant(
hidden_states,
norm.weight,
@@ -224,9 +230,11 @@ def _update_and_read_residual_aiter_fp8_group(
dtype_quant=torch.float8_e4m3fn,
res1=residual,
output_unquantized_inp1=needs_bf16,
transpose_scale=False,
transpose_scale=emit_transposed_scale,
)
if _use_aiter_bpreshuffle_gfx95:
if emit_transposed_scale:
output = view_aiter_fused_rms_transposed_fp8_scale_tuple(output)
elif _use_aiter_bpreshuffle_gfx95:
output = materialize_bpreshuffle_fp8_scale_tuple(output)
if needs_bf16:
output = (output[0], output[1], unquantized)
@@ -194,7 +194,7 @@ def emit_transposed_bpreshuffle_scale(m: int, *, on_bpreshuffle_gfx95: bool) ->
zero-copy path is only taken on gfx95 bpreshuffle and only for M(tokens) >= 2:
at M == 1 the ``[1, G]`` and ``[G, 1]`` byte orders coincide, so the transposed
emit buys nothing and the materialize path is used. Centralizes the gate shared
by the MoE-down and MLA o_proj producer sites.
by the gfx95 bpreshuffle producer sites.
"""
return on_bpreshuffle_gfx95 and m >= 2
@@ -16,7 +16,9 @@ from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton
from sglang.srt.layers.dcp import all_gather_kv_cache_for_mha_extend
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.layers.quantization.fp8_utils import (
emit_transposed_bpreshuffle_scale,
materialize_bpreshuffle_fp8_scale_tuple,
view_aiter_fused_rms_transposed_fp8_scale_tuple,
)
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
@@ -75,6 +77,10 @@ class DeepseekMHARocmForwardMixin:
# the unquantized output for q_lora; otherwise q_lora becomes the (fp8,scale)
# tuple.
if _use_aiter_gfx95 and _is_block_scale_fp8(self.q_b_proj):
emit_transposed_scale = emit_transposed_bpreshuffle_scale(
q.shape[0],
on_bpreshuffle_gfx95=_use_aiter_bpreshuffle_gfx95,
)
q_quanted, q_lora, _, _ = fused_rms_fp8_group_quant(
q,
self.q_a_layernorm.weight,
@@ -86,9 +92,13 @@ class DeepseekMHARocmForwardMixin:
dtype_quant=torch.float8_e4m3fn,
res1=None,
output_unquantized_inp1=True,
transpose_scale=False,
transpose_scale=emit_transposed_scale,
)
if _use_aiter_bpreshuffle_gfx95:
if emit_transposed_scale:
q_quanted = view_aiter_fused_rms_transposed_fp8_scale_tuple(
q_quanted
)
elif _use_aiter_bpreshuffle_gfx95:
q_quanted = materialize_bpreshuffle_fp8_scale_tuple(q_quanted)
q = self.q_b_proj(q_quanted)[0].view(
-1, self.num_local_heads, self.qk_head_dim
@@ -119,6 +129,10 @@ class DeepseekMHARocmForwardMixin:
)
q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, self.qk_head_dim)
elif _use_aiter_gfx95 and _is_block_scale_fp8(self.q_b_proj):
emit_transposed_scale = emit_transposed_bpreshuffle_scale(
q.shape[0],
on_bpreshuffle_gfx95=_use_aiter_bpreshuffle_gfx95,
)
q, _, _, _ = fused_rms_fp8_group_quant(
q,
self.q_a_layernorm.weight,
@@ -130,9 +144,11 @@ class DeepseekMHARocmForwardMixin:
dtype_quant=torch.float8_e4m3fn,
res1=None,
output_unquantized_inp1=False,
transpose_scale=False,
transpose_scale=emit_transposed_scale,
)
if _use_aiter_bpreshuffle_gfx95:
if emit_transposed_scale:
q = view_aiter_fused_rms_transposed_fp8_scale_tuple(q)
elif _use_aiter_bpreshuffle_gfx95:
q = materialize_bpreshuffle_fp8_scale_tuple(q)
q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, self.qk_head_dim)
else:
@@ -150,6 +166,10 @@ class DeepseekMHARocmForwardMixin:
latent_cache = latent_cache.unsqueeze(1)
if _use_aiter_gfx95 and _is_block_scale_fp8(self.kv_b_proj):
emit_transposed_scale = emit_transposed_bpreshuffle_scale(
kv_a.shape[0],
on_bpreshuffle_gfx95=_use_aiter_bpreshuffle_gfx95,
)
kv_a_quanted, kv_a, _, _ = fused_rms_fp8_group_quant(
kv_a,
self.kv_a_layernorm.weight,
@@ -161,9 +181,13 @@ class DeepseekMHARocmForwardMixin:
dtype_quant=torch.float8_e4m3fn,
res1=None,
output_unquantized_inp1=True, # return unqaunt kv_a
transpose_scale=False,
transpose_scale=emit_transposed_scale,
)
if _use_aiter_bpreshuffle_gfx95:
if emit_transposed_scale:
kv_a_quanted = view_aiter_fused_rms_transposed_fp8_scale_tuple(
kv_a_quanted
)
elif _use_aiter_bpreshuffle_gfx95:
kv_a_quanted = materialize_bpreshuffle_fp8_scale_tuple(kv_a_quanted)
else:
kv_a = self.kv_a_layernorm(kv_a)
@@ -540,6 +540,10 @@ class DeepseekMLARocmForwardMixin:
self.kv_a_layernorm.variance_epsilon,
)
elif _use_aiter_gfx95 and _is_block_scale_fp8(self.q_b_proj):
emit_transposed_scale = emit_transposed_bpreshuffle_scale(
q.shape[0],
on_bpreshuffle_gfx95=_use_aiter_bpreshuffle_gfx95,
)
if self.use_dsa:
q_quanted, q_lora, k_nope, _ = fused_rms_fp8_group_quant(
q,
@@ -552,9 +556,13 @@ class DeepseekMLARocmForwardMixin:
dtype_quant=torch.float8_e4m3fn,
res1=None,
output_unquantized_inp1=True,
transpose_scale=False,
transpose_scale=emit_transposed_scale,
)
if _use_aiter_bpreshuffle_gfx95:
if emit_transposed_scale:
q_quanted = view_aiter_fused_rms_transposed_fp8_scale_tuple(
q_quanted
)
elif _use_aiter_bpreshuffle_gfx95:
q_quanted = materialize_bpreshuffle_fp8_scale_tuple(q_quanted)
q = q_quanted
else:
@@ -569,9 +577,11 @@ class DeepseekMLARocmForwardMixin:
dtype_quant=torch.float8_e4m3fn,
res1=None,
output_unquantized_inp1=False,
transpose_scale=False,
transpose_scale=emit_transposed_scale,
)
if _use_aiter_bpreshuffle_gfx95:
if emit_transposed_scale:
q = view_aiter_fused_rms_transposed_fp8_scale_tuple(q)
elif _use_aiter_bpreshuffle_gfx95:
q = materialize_bpreshuffle_fp8_scale_tuple(q)
elif _use_aiter:
q, k_nope = fused_qk_rmsnorm_bf16(