[Perf] dsv41: Feed wo_b the fused MXFP8 attention output in the prefill graph (#46231)

This commit is contained in:
Liangsheng Yin
2026-10-23 21:31:33 -07:00
committed by GitHub
parent a7183a8246
commit 564aec61c8
2 changed files with 63 additions and 15 deletions
+52 -8
View File
@@ -201,6 +201,7 @@ from sglang.srt.runtime_context import (
from sglang.srt.utils import (
LazyValue,
add_prefix,
ceil_align,
get_bool_env_var,
is_gfx95_supported,
is_gfx942_supported,
@@ -833,6 +834,38 @@ bcg_deepseek_v4_mega_attention_with_output = eager_on_graph(True)(
)
def deepseek_v4_mega_attention_with_mxfp8_output(
layer, query, data, scale, attn_sink
) -> None:
forward_batch = get_tc_piecewise_forward_context().forward_batch
real_num_tokens = forward_batch.global_num_token_non_padded_cpu
if real_num_tokens:
# DeepGEMM writes the scales at the TMA-aligned stride of the live rows,
# while the captured wo_b reads them at the stride of the padded rows.
restride = scale.stride(-1) != ceil_align(real_num_tokens, 4)
live_scale = (
mhc.empty_deepgemm_mxfp8_scales(real_num_tokens, data.shape[1], data.device)
if restride
else scale[:real_num_tokens]
)
layer._mega_core(
query[:real_num_tokens],
attn_sink,
forward_batch,
get_attn_backend(),
out=(data[:real_num_tokens], live_scale),
)
if restride:
scale[:real_num_tokens].copy_(live_scale)
data[real_num_tokens:].zero_()
scale[real_num_tokens:].zero_()
bcg_deepseek_v4_mega_attention_with_mxfp8_output = eager_on_graph(True)(
deepseek_v4_mega_attention_with_mxfp8_output
)
def deepseek_v4_low_ratio_sources(layer, x, q_lora, positions) -> None:
# The compressor and prefill indexer sync with the host, like the attention.
forward_batch = get_tc_piecewise_forward_context().forward_batch
@@ -1553,13 +1586,25 @@ class MQALayer(MqaAttentionBase):
if not self.mega_wq_b_permuted:
q = pack_query(q, kernel_num_heads)
if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph():
o = q.new_empty((q.shape[0], self.n_local_groups, self.o_lora_rank))
rows = q.shape[0]
if self._mega_mxfp8_output(rows):
y, y_sf = mhc.empty_deepgemm_mxfp8_pair(
rows, self.n_local_groups * self.o_lora_rank, q.device
)
bcg_deepseek_v4_mega_attention_with_mxfp8_output(
self, q, y, y_sf, attn_sink
)
return y, y_sf
o = q.new_empty((rows, self.n_local_groups, self.o_lora_rank))
bcg_deepseek_v4_mega_attention_with_output(self, q, o, attn_sink)
return o
return self._mega_core(
q, attn_sink, forward_batch, attn_backend, allow_mxfp8_output=True
)
def _mega_mxfp8_output(self, rows: int) -> bool:
return self.wo_b.deepgemm_mxfp8_input and mhc.fused_mxfp8_rows(rows)
def _mega_core(
self,
q: torch.Tensor,
@@ -1567,8 +1612,9 @@ class MQALayer(MqaAttentionBase):
forward_batch: ForwardBatch,
attn_backend,
allow_mxfp8_output: bool = False,
out: Optional[torch.Tensor] = None,
out: Optional[torch.Tensor | Tuple[torch.Tensor, torch.Tensor]] = None,
) -> torch.Tensor | Tuple[torch.Tensor, torch.Tensor]:
"""``out`` is a bf16 tensor or an (fp8, DeepGEMM-packed scale) pair."""
data, scale = attn_backend.forward(
q=q,
k=q,
@@ -1583,14 +1629,12 @@ class MQALayer(MqaAttentionBase):
groups = self.n_local_groups
rows = data.shape[0]
o_fp8, o_s = data[:, :groups], scale[:, :groups]
if (
allow_mxfp8_output
and self.wo_b.deepgemm_mxfp8_input
and mhc.fused_mxfp8_rows(rows)
):
y, y_sf = mhc.empty_deepgemm_mxfp8_pair(
if out is None and allow_mxfp8_output and self._mega_mxfp8_output(rows):
out = mhc.empty_deepgemm_mxfp8_pair(
rows, groups * self.o_lora_rank, data.device
)
if isinstance(out, tuple):
y, y_sf = out
_fp8_wo_a_einsum(
o_fp8,
o_s,
+11 -7
View File
@@ -68,12 +68,9 @@ def fused_mxfp8_rows(rows: int) -> bool:
return rows <= FUSED_MXFP8_PRODUCER_MAX_ROWS
def empty_deepgemm_mxfp8_pair(
rows: int, cols: int, device
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Empty fp8 rows and DeepGEMM-packed UE8M0 scales for a producer to fill."""
data = torch.empty((rows, cols), device=device, dtype=torch.float8_e4m3fn)
scales = create_per_token_group_quant_fp8_output_scale(
def empty_deepgemm_mxfp8_scales(rows: int, cols: int, device) -> torch.Tensor:
"""Empty DeepGEMM-packed UE8M0 scales for ``rows`` x ``cols`` fp8 data."""
return create_per_token_group_quant_fp8_output_scale(
(rows, cols),
device,
32,
@@ -81,7 +78,14 @@ def empty_deepgemm_mxfp8_pair(
scale_tma_aligned=True,
scale_ue8m0=True,
)
return data, scales
def empty_deepgemm_mxfp8_pair(
rows: int, cols: int, device
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Empty fp8 rows and DeepGEMM-packed UE8M0 scales for a producer to fill."""
data = torch.empty((rows, cols), device=device, dtype=torch.float8_e4m3fn)
return data, empty_deepgemm_mxfp8_scales(rows, cols, device)
def fused_mhc_tp_ok(tp_size: int) -> bool: