From 564aec61c8b138225ba03bb823ecd1d457b5699a Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 23 Oct 2026 21:31:33 -0700 Subject: [PATCH] [Perf] dsv41: Feed wo_b the fused MXFP8 attention output in the prefill graph (#46231) --- python/sglang/srt/models/deepseek_v4.py | 60 ++++++++++++++++++--- python/sglang/srt/models/deepseek_v4_mhc.py | 18 ++++--- 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 41ceec4485..76473c12bc 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -201,6 +201,7 @@ from sglang.srt.runtime_context import ( from sglang.srt.utils import ( LazyValue, add_prefix, + ceil_align, get_bool_env_var, is_gfx95_supported, is_gfx942_supported, @@ -833,6 +834,38 @@ bcg_deepseek_v4_mega_attention_with_output = eager_on_graph(True)( ) +def deepseek_v4_mega_attention_with_mxfp8_output( + layer, query, data, scale, attn_sink +) -> None: + forward_batch = get_tc_piecewise_forward_context().forward_batch + real_num_tokens = forward_batch.global_num_token_non_padded_cpu + if real_num_tokens: + # DeepGEMM writes the scales at the TMA-aligned stride of the live rows, + # while the captured wo_b reads them at the stride of the padded rows. + restride = scale.stride(-1) != ceil_align(real_num_tokens, 4) + live_scale = ( + mhc.empty_deepgemm_mxfp8_scales(real_num_tokens, data.shape[1], data.device) + if restride + else scale[:real_num_tokens] + ) + layer._mega_core( + query[:real_num_tokens], + attn_sink, + forward_batch, + get_attn_backend(), + out=(data[:real_num_tokens], live_scale), + ) + if restride: + scale[:real_num_tokens].copy_(live_scale) + data[real_num_tokens:].zero_() + scale[real_num_tokens:].zero_() + + +bcg_deepseek_v4_mega_attention_with_mxfp8_output = eager_on_graph(True)( + deepseek_v4_mega_attention_with_mxfp8_output +) + + def deepseek_v4_low_ratio_sources(layer, x, q_lora, positions) -> None: # The compressor and prefill indexer sync with the host, like the attention. forward_batch = get_tc_piecewise_forward_context().forward_batch @@ -1553,13 +1586,25 @@ class MQALayer(MqaAttentionBase): if not self.mega_wq_b_permuted: q = pack_query(q, kernel_num_heads) if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph(): - o = q.new_empty((q.shape[0], self.n_local_groups, self.o_lora_rank)) + rows = q.shape[0] + if self._mega_mxfp8_output(rows): + y, y_sf = mhc.empty_deepgemm_mxfp8_pair( + rows, self.n_local_groups * self.o_lora_rank, q.device + ) + bcg_deepseek_v4_mega_attention_with_mxfp8_output( + self, q, y, y_sf, attn_sink + ) + return y, y_sf + o = q.new_empty((rows, self.n_local_groups, self.o_lora_rank)) bcg_deepseek_v4_mega_attention_with_output(self, q, o, attn_sink) return o return self._mega_core( q, attn_sink, forward_batch, attn_backend, allow_mxfp8_output=True ) + def _mega_mxfp8_output(self, rows: int) -> bool: + return self.wo_b.deepgemm_mxfp8_input and mhc.fused_mxfp8_rows(rows) + def _mega_core( self, q: torch.Tensor, @@ -1567,8 +1612,9 @@ class MQALayer(MqaAttentionBase): forward_batch: ForwardBatch, attn_backend, allow_mxfp8_output: bool = False, - out: Optional[torch.Tensor] = None, + out: Optional[torch.Tensor | Tuple[torch.Tensor, torch.Tensor]] = None, ) -> torch.Tensor | Tuple[torch.Tensor, torch.Tensor]: + """``out`` is a bf16 tensor or an (fp8, DeepGEMM-packed scale) pair.""" data, scale = attn_backend.forward( q=q, k=q, @@ -1583,14 +1629,12 @@ class MQALayer(MqaAttentionBase): groups = self.n_local_groups rows = data.shape[0] o_fp8, o_s = data[:, :groups], scale[:, :groups] - if ( - allow_mxfp8_output - and self.wo_b.deepgemm_mxfp8_input - and mhc.fused_mxfp8_rows(rows) - ): - y, y_sf = mhc.empty_deepgemm_mxfp8_pair( + if out is None and allow_mxfp8_output and self._mega_mxfp8_output(rows): + out = mhc.empty_deepgemm_mxfp8_pair( rows, groups * self.o_lora_rank, data.device ) + if isinstance(out, tuple): + y, y_sf = out _fp8_wo_a_einsum( o_fp8, o_s, diff --git a/python/sglang/srt/models/deepseek_v4_mhc.py b/python/sglang/srt/models/deepseek_v4_mhc.py index 38a68c80e4..b8ff7c473e 100644 --- a/python/sglang/srt/models/deepseek_v4_mhc.py +++ b/python/sglang/srt/models/deepseek_v4_mhc.py @@ -68,12 +68,9 @@ def fused_mxfp8_rows(rows: int) -> bool: return rows <= FUSED_MXFP8_PRODUCER_MAX_ROWS -def empty_deepgemm_mxfp8_pair( - rows: int, cols: int, device -) -> Tuple[torch.Tensor, torch.Tensor]: - """Empty fp8 rows and DeepGEMM-packed UE8M0 scales for a producer to fill.""" - data = torch.empty((rows, cols), device=device, dtype=torch.float8_e4m3fn) - scales = create_per_token_group_quant_fp8_output_scale( +def empty_deepgemm_mxfp8_scales(rows: int, cols: int, device) -> torch.Tensor: + """Empty DeepGEMM-packed UE8M0 scales for ``rows`` x ``cols`` fp8 data.""" + return create_per_token_group_quant_fp8_output_scale( (rows, cols), device, 32, @@ -81,7 +78,14 @@ def empty_deepgemm_mxfp8_pair( scale_tma_aligned=True, scale_ue8m0=True, ) - return data, scales + + +def empty_deepgemm_mxfp8_pair( + rows: int, cols: int, device +) -> Tuple[torch.Tensor, torch.Tensor]: + """Empty fp8 rows and DeepGEMM-packed UE8M0 scales for a producer to fill.""" + data = torch.empty((rows, cols), device=device, dtype=torch.float8_e4m3fn) + return data, empty_deepgemm_mxfp8_scales(rows, cols, device) def fused_mhc_tp_ok(tp_size: int) -> bool: