[AMD] Eliminate remaining gfx95 FP8 scale relayout copies (#41030)
This commit is contained in:
@@ -33,7 +33,9 @@ from sglang.srt.layers.layer_boundary.adapters.attention import (
|
||||
from sglang.srt.layers.layer_boundary.residual import LayerResidualOps
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
_use_aiter_bpreshuffle_gfx95,
|
||||
emit_transposed_bpreshuffle_scale,
|
||||
materialize_bpreshuffle_fp8_scale_tuple,
|
||||
view_aiter_fused_rms_transposed_fp8_scale_tuple,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_platform
|
||||
@@ -213,6 +215,10 @@ def _update_and_read_residual_aiter_fp8_group(
|
||||
unquantized bf16 output rides along as a third element, so the DSA indexer
|
||||
can skip dequantizing. post_residual_addition is not applied on this path."""
|
||||
needs_bf16 = get_attn_tp_context().is_dsa
|
||||
emit_transposed_scale = emit_transposed_bpreshuffle_scale(
|
||||
hidden_states.shape[0],
|
||||
on_bpreshuffle_gfx95=_use_aiter_bpreshuffle_gfx95,
|
||||
)
|
||||
output, unquantized, _, residual_out = fused_rms_fp8_group_quant(
|
||||
hidden_states,
|
||||
norm.weight,
|
||||
@@ -224,9 +230,11 @@ def _update_and_read_residual_aiter_fp8_group(
|
||||
dtype_quant=torch.float8_e4m3fn,
|
||||
res1=residual,
|
||||
output_unquantized_inp1=needs_bf16,
|
||||
transpose_scale=False,
|
||||
transpose_scale=emit_transposed_scale,
|
||||
)
|
||||
if _use_aiter_bpreshuffle_gfx95:
|
||||
if emit_transposed_scale:
|
||||
output = view_aiter_fused_rms_transposed_fp8_scale_tuple(output)
|
||||
elif _use_aiter_bpreshuffle_gfx95:
|
||||
output = materialize_bpreshuffle_fp8_scale_tuple(output)
|
||||
if needs_bf16:
|
||||
output = (output[0], output[1], unquantized)
|
||||
|
||||
@@ -194,7 +194,7 @@ def emit_transposed_bpreshuffle_scale(m: int, *, on_bpreshuffle_gfx95: bool) ->
|
||||
zero-copy path is only taken on gfx95 bpreshuffle and only for M(tokens) >= 2:
|
||||
at M == 1 the ``[1, G]`` and ``[G, 1]`` byte orders coincide, so the transposed
|
||||
emit buys nothing and the materialize path is used. Centralizes the gate shared
|
||||
by the MoE-down and MLA o_proj producer sites.
|
||||
by the gfx95 bpreshuffle producer sites.
|
||||
"""
|
||||
return on_bpreshuffle_gfx95 and m >= 2
|
||||
|
||||
|
||||
+30
-6
@@ -16,7 +16,9 @@ from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton
|
||||
from sglang.srt.layers.dcp import all_gather_kv_cache_for_mha_extend
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
emit_transposed_bpreshuffle_scale,
|
||||
materialize_bpreshuffle_fp8_scale_tuple,
|
||||
view_aiter_fused_rms_transposed_fp8_scale_tuple,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -75,6 +77,10 @@ class DeepseekMHARocmForwardMixin:
|
||||
# the unquantized output for q_lora; otherwise q_lora becomes the (fp8,scale)
|
||||
# tuple.
|
||||
if _use_aiter_gfx95 and _is_block_scale_fp8(self.q_b_proj):
|
||||
emit_transposed_scale = emit_transposed_bpreshuffle_scale(
|
||||
q.shape[0],
|
||||
on_bpreshuffle_gfx95=_use_aiter_bpreshuffle_gfx95,
|
||||
)
|
||||
q_quanted, q_lora, _, _ = fused_rms_fp8_group_quant(
|
||||
q,
|
||||
self.q_a_layernorm.weight,
|
||||
@@ -86,9 +92,13 @@ class DeepseekMHARocmForwardMixin:
|
||||
dtype_quant=torch.float8_e4m3fn,
|
||||
res1=None,
|
||||
output_unquantized_inp1=True,
|
||||
transpose_scale=False,
|
||||
transpose_scale=emit_transposed_scale,
|
||||
)
|
||||
if _use_aiter_bpreshuffle_gfx95:
|
||||
if emit_transposed_scale:
|
||||
q_quanted = view_aiter_fused_rms_transposed_fp8_scale_tuple(
|
||||
q_quanted
|
||||
)
|
||||
elif _use_aiter_bpreshuffle_gfx95:
|
||||
q_quanted = materialize_bpreshuffle_fp8_scale_tuple(q_quanted)
|
||||
q = self.q_b_proj(q_quanted)[0].view(
|
||||
-1, self.num_local_heads, self.qk_head_dim
|
||||
@@ -119,6 +129,10 @@ class DeepseekMHARocmForwardMixin:
|
||||
)
|
||||
q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
elif _use_aiter_gfx95 and _is_block_scale_fp8(self.q_b_proj):
|
||||
emit_transposed_scale = emit_transposed_bpreshuffle_scale(
|
||||
q.shape[0],
|
||||
on_bpreshuffle_gfx95=_use_aiter_bpreshuffle_gfx95,
|
||||
)
|
||||
q, _, _, _ = fused_rms_fp8_group_quant(
|
||||
q,
|
||||
self.q_a_layernorm.weight,
|
||||
@@ -130,9 +144,11 @@ class DeepseekMHARocmForwardMixin:
|
||||
dtype_quant=torch.float8_e4m3fn,
|
||||
res1=None,
|
||||
output_unquantized_inp1=False,
|
||||
transpose_scale=False,
|
||||
transpose_scale=emit_transposed_scale,
|
||||
)
|
||||
if _use_aiter_bpreshuffle_gfx95:
|
||||
if emit_transposed_scale:
|
||||
q = view_aiter_fused_rms_transposed_fp8_scale_tuple(q)
|
||||
elif _use_aiter_bpreshuffle_gfx95:
|
||||
q = materialize_bpreshuffle_fp8_scale_tuple(q)
|
||||
q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
else:
|
||||
@@ -150,6 +166,10 @@ class DeepseekMHARocmForwardMixin:
|
||||
latent_cache = latent_cache.unsqueeze(1)
|
||||
|
||||
if _use_aiter_gfx95 and _is_block_scale_fp8(self.kv_b_proj):
|
||||
emit_transposed_scale = emit_transposed_bpreshuffle_scale(
|
||||
kv_a.shape[0],
|
||||
on_bpreshuffle_gfx95=_use_aiter_bpreshuffle_gfx95,
|
||||
)
|
||||
kv_a_quanted, kv_a, _, _ = fused_rms_fp8_group_quant(
|
||||
kv_a,
|
||||
self.kv_a_layernorm.weight,
|
||||
@@ -161,9 +181,13 @@ class DeepseekMHARocmForwardMixin:
|
||||
dtype_quant=torch.float8_e4m3fn,
|
||||
res1=None,
|
||||
output_unquantized_inp1=True, # return unqaunt kv_a
|
||||
transpose_scale=False,
|
||||
transpose_scale=emit_transposed_scale,
|
||||
)
|
||||
if _use_aiter_bpreshuffle_gfx95:
|
||||
if emit_transposed_scale:
|
||||
kv_a_quanted = view_aiter_fused_rms_transposed_fp8_scale_tuple(
|
||||
kv_a_quanted
|
||||
)
|
||||
elif _use_aiter_bpreshuffle_gfx95:
|
||||
kv_a_quanted = materialize_bpreshuffle_fp8_scale_tuple(kv_a_quanted)
|
||||
else:
|
||||
kv_a = self.kv_a_layernorm(kv_a)
|
||||
|
||||
+14
-4
@@ -540,6 +540,10 @@ class DeepseekMLARocmForwardMixin:
|
||||
self.kv_a_layernorm.variance_epsilon,
|
||||
)
|
||||
elif _use_aiter_gfx95 and _is_block_scale_fp8(self.q_b_proj):
|
||||
emit_transposed_scale = emit_transposed_bpreshuffle_scale(
|
||||
q.shape[0],
|
||||
on_bpreshuffle_gfx95=_use_aiter_bpreshuffle_gfx95,
|
||||
)
|
||||
if self.use_dsa:
|
||||
q_quanted, q_lora, k_nope, _ = fused_rms_fp8_group_quant(
|
||||
q,
|
||||
@@ -552,9 +556,13 @@ class DeepseekMLARocmForwardMixin:
|
||||
dtype_quant=torch.float8_e4m3fn,
|
||||
res1=None,
|
||||
output_unquantized_inp1=True,
|
||||
transpose_scale=False,
|
||||
transpose_scale=emit_transposed_scale,
|
||||
)
|
||||
if _use_aiter_bpreshuffle_gfx95:
|
||||
if emit_transposed_scale:
|
||||
q_quanted = view_aiter_fused_rms_transposed_fp8_scale_tuple(
|
||||
q_quanted
|
||||
)
|
||||
elif _use_aiter_bpreshuffle_gfx95:
|
||||
q_quanted = materialize_bpreshuffle_fp8_scale_tuple(q_quanted)
|
||||
q = q_quanted
|
||||
else:
|
||||
@@ -569,9 +577,11 @@ class DeepseekMLARocmForwardMixin:
|
||||
dtype_quant=torch.float8_e4m3fn,
|
||||
res1=None,
|
||||
output_unquantized_inp1=False,
|
||||
transpose_scale=False,
|
||||
transpose_scale=emit_transposed_scale,
|
||||
)
|
||||
if _use_aiter_bpreshuffle_gfx95:
|
||||
if emit_transposed_scale:
|
||||
q = view_aiter_fused_rms_transposed_fp8_scale_tuple(q)
|
||||
elif _use_aiter_bpreshuffle_gfx95:
|
||||
q = materialize_bpreshuffle_fp8_scale_tuple(q)
|
||||
elif _use_aiter:
|
||||
q, k_nope = fused_qk_rmsnorm_bf16(
|
||||
|
||||
Reference in New Issue
Block a user