From da4bf1ef8ebc8af119b3f7740b0d7a3ee281055a Mon Sep 17 00:00:00 2001 From: Chunan Zeng Date: Mon, 12 Oct 2026 00:27:22 -0700 Subject: [PATCH] [Perf] Route BF16 mixed-batch decode tails through decode attention (#43238) Co-authored-by: Claude Opus 5.5 Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- .../layers/attention/trtllm_mha_backend.py | 72 ++++++++++++++++--- 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/python/sglang/srt/layers/attention/trtllm_mha_backend.py b/python/sglang/srt/layers/attention/trtllm_mha_backend.py index 204fbb169d..630cab92f6 100644 --- a/python/sglang/srt/layers/attention/trtllm_mha_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mha_backend.py @@ -158,6 +158,9 @@ class TRTLLMMHAMetadata: encoder_cache_seqlens: torch.Tensor = None encoder_page_table: torch.Tensor = None encoder_row_map: torch.Tensor = None + # Prefix before trailing one-token requests in an extend-shaped batch. + mixed_prefill_reqs: int = 0 + mixed_prefill_tokens: int = 0 class TRTLLMHAAttnBackend(FlashInferAttnBackend): @@ -1258,6 +1261,17 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): # Query-side max length, sourced from the host-resident extend lengths # (sync-free); for plain prefill these equal the full seq lens. metadata.max_seq_len_q = int(max(forward_batch.extend_seq_lens_cpu)) + if ( + forward_batch.forward_mode.is_extend() + and forward_batch.spec_info is None + ): + lengths = forward_batch.extend_seq_lens_cpu + prefix = len(lengths) + while prefix and lengths[prefix - 1] == 1: + prefix -= 1 + if 0 < prefix < len(lengths): + metadata.mixed_prefill_reqs = prefix + metadata.mixed_prefill_tokens = sum(lengths[:prefix]) if ( forward_batch.extend_prefix_lens_cpu is not None and any(forward_batch.extend_prefix_lens_cpu) @@ -1784,8 +1798,11 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): cu_seqlens_kv, use_zigzag_page_table=False, out=None, + page_table_override=None, ): - block_tables = page_table + block_tables = ( + page_table if page_table_override is None else page_table_override + ) if use_zigzag_page_table: block_tables = self.forward_metadata.zigzag_page_table zigzag_swa_pt = self.forward_metadata.zigzag_swa_page_table @@ -1829,14 +1846,51 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): out = native_out if uses_native_fp4 else forward_batch._attn_output if out is not None: out = out.view_as(q) - o = _trtllm_context_attn( - q, - self.forward_metadata.cu_seqlens_q, - self.forward_metadata.cache_seqlens_int32, - self.forward_metadata.max_seq_len_q, - cu_seqlens_kv=self.forward_metadata.cu_seqlens_k, - out=out, - ) + prefix_reqs = self.forward_metadata.mixed_prefill_reqs + prefix_tokens = self.forward_metadata.mixed_prefill_tokens + if ( + prefix_reqs + and self.data_type in (torch.float16, torch.bfloat16) + and q.dtype == self.data_type + and layer.attn_type == AttentionType.DECODER + ): + out = torch.empty_like(q) if out is None else out + _trtllm_context_attn( + q[:prefix_tokens], + self.forward_metadata.cu_seqlens_q[: prefix_reqs + 1], + self.forward_metadata.cache_seqlens_int32[:prefix_reqs], + self.forward_metadata.max_seq_len_q, + cu_seqlens_kv=self.forward_metadata.cu_seqlens_k[ + : prefix_reqs + 1 + ], + out=out[:prefix_tokens], + page_table_override=page_table[:prefix_reqs], + ) + # Eager DP batches can pad Q without adding requests. + tail_end = prefix_tokens + ( + self.forward_metadata.cache_seqlens_int32.shape[0] - prefix_reqs + ) + self._run_fixed_q_len_decode( + q[prefix_tokens:tail_end], + kv_cache, + page_table[prefix_reqs:], + self.forward_metadata.cache_seqlens_int32[prefix_reqs:], + bmm1_scale=bmm1_scale, + bmm2_scale=bmm2_scale, + window_left=layer.sliding_window_size, + sinks=attention_sink, + out=out[prefix_tokens:tail_end], + ) + o = out + else: + o = _trtllm_context_attn( + q, + self.forward_metadata.cu_seqlens_q, + self.forward_metadata.cache_seqlens_int32, + self.forward_metadata.max_seq_len_q, + cu_seqlens_kv=self.forward_metadata.cu_seqlens_k, + out=out, + ) if uses_native_fp4: o = self._finalize_nvfp4_output(o, forward_batch)