[Perf] Route BF16 mixed-batch decode tails through decode attention (#43238)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
Chunan Zeng
2026-10-12 15:27:22 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 Xiaoyu Zhang
parent a0e6effe50
commit da4bf1ef8e
@@ -158,6 +158,9 @@ class TRTLLMMHAMetadata:
encoder_cache_seqlens: torch.Tensor = None
encoder_page_table: torch.Tensor = None
encoder_row_map: torch.Tensor = None
# Prefix before trailing one-token requests in an extend-shaped batch.
mixed_prefill_reqs: int = 0
mixed_prefill_tokens: int = 0
class TRTLLMHAAttnBackend(FlashInferAttnBackend):
@@ -1258,6 +1261,17 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
# Query-side max length, sourced from the host-resident extend lengths
# (sync-free); for plain prefill these equal the full seq lens.
metadata.max_seq_len_q = int(max(forward_batch.extend_seq_lens_cpu))
if (
forward_batch.forward_mode.is_extend()
and forward_batch.spec_info is None
):
lengths = forward_batch.extend_seq_lens_cpu
prefix = len(lengths)
while prefix and lengths[prefix - 1] == 1:
prefix -= 1
if 0 < prefix < len(lengths):
metadata.mixed_prefill_reqs = prefix
metadata.mixed_prefill_tokens = sum(lengths[:prefix])
if (
forward_batch.extend_prefix_lens_cpu is not None
and any(forward_batch.extend_prefix_lens_cpu)
@@ -1784,8 +1798,11 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
cu_seqlens_kv,
use_zigzag_page_table=False,
out=None,
page_table_override=None,
):
block_tables = page_table
block_tables = (
page_table if page_table_override is None else page_table_override
)
if use_zigzag_page_table:
block_tables = self.forward_metadata.zigzag_page_table
zigzag_swa_pt = self.forward_metadata.zigzag_swa_page_table
@@ -1829,14 +1846,51 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
out = native_out if uses_native_fp4 else forward_batch._attn_output
if out is not None:
out = out.view_as(q)
o = _trtllm_context_attn(
q,
self.forward_metadata.cu_seqlens_q,
self.forward_metadata.cache_seqlens_int32,
self.forward_metadata.max_seq_len_q,
cu_seqlens_kv=self.forward_metadata.cu_seqlens_k,
out=out,
)
prefix_reqs = self.forward_metadata.mixed_prefill_reqs
prefix_tokens = self.forward_metadata.mixed_prefill_tokens
if (
prefix_reqs
and self.data_type in (torch.float16, torch.bfloat16)
and q.dtype == self.data_type
and layer.attn_type == AttentionType.DECODER
):
out = torch.empty_like(q) if out is None else out
_trtllm_context_attn(
q[:prefix_tokens],
self.forward_metadata.cu_seqlens_q[: prefix_reqs + 1],
self.forward_metadata.cache_seqlens_int32[:prefix_reqs],
self.forward_metadata.max_seq_len_q,
cu_seqlens_kv=self.forward_metadata.cu_seqlens_k[
: prefix_reqs + 1
],
out=out[:prefix_tokens],
page_table_override=page_table[:prefix_reqs],
)
# Eager DP batches can pad Q without adding requests.
tail_end = prefix_tokens + (
self.forward_metadata.cache_seqlens_int32.shape[0] - prefix_reqs
)
self._run_fixed_q_len_decode(
q[prefix_tokens:tail_end],
kv_cache,
page_table[prefix_reqs:],
self.forward_metadata.cache_seqlens_int32[prefix_reqs:],
bmm1_scale=bmm1_scale,
bmm2_scale=bmm2_scale,
window_left=layer.sliding_window_size,
sinks=attention_sink,
out=out[prefix_tokens:tail_end],
)
o = out
else:
o = _trtllm_context_attn(
q,
self.forward_metadata.cu_seqlens_q,
self.forward_metadata.cache_seqlens_int32,
self.forward_metadata.max_seq_len_q,
cu_seqlens_kv=self.forward_metadata.cu_seqlens_k,
out=out,
)
if uses_native_fp4:
o = self._finalize_nvfp4_output(o, forward_batch)