Fix XQA draft extend with FP8 KV cache in trtllm_mha (#41656)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
elvischenv
2026-10-05 01:37:54 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent abd6c374c7
commit 55dcf3254f
2 changed files with 73 additions and 42 deletions
@@ -1600,15 +1600,17 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
*self._kv_write_scales(layer),
)
uses_decode_kernel = (
forward_batch.forward_mode.is_target_verify()
or forward_batch.forward_mode.is_draft_extend_v2()
)
q_scale = 1.0
if uses_native_fp4:
q = q.to(torch.float8_e4m3fn)
elif (
self.data_type == torch.float8_e4m3fn
and (
not self.is_xqa_impl
or not forward_batch.forward_mode.is_target_verify()
)
and (not self.is_xqa_impl or not uses_decode_kernel)
and not use_fused_qkv
):
q = q.to(torch.float8_e4m3fn)
@@ -1617,11 +1619,6 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
else:
q = q.reshape(-1, layer.tp_q_head_num, layer.head_dim)
is_decode_mode = (
forward_batch.forward_mode.is_target_verify()
or forward_batch.forward_mode.is_draft_extend_v2()
)
if uses_native_fp4:
kv_cache, kv_cache_block_scales = self._get_nvfp4_decode_kv_cache(layer)
k_cache, v_cache = kv_cache
@@ -1631,7 +1628,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
k_cache_raw, v_cache_raw = self.token_to_kv_pool.get_kv_buffer(
layer.layer_id
)
if not self.use_fmha_v2 or is_decode_mode:
if not self.use_fmha_v2 or uses_decode_kernel:
# Decode and SM100 batch_context kernels require HND layout.
k_cache, v_cache = self._reshape_paged_kv_cache(
k_cache_raw, v_cache_raw, layer, layer.head_dim
@@ -1658,7 +1655,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
page_table = self._get_layer_page_table(layer, forward_batch)
native_out = self._nvfp4_output_view(q) if uses_native_fp4 else None
if is_decode_mode:
if uses_decode_kernel:
if (
forward_batch.forward_mode.is_target_verify()
and layer.attn_type == AttentionType.ENCODER_ONLY
@@ -1716,8 +1713,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
else:
mask = (
self._xqa_spec_dec_mask
if forward_batch.forward_mode.is_target_verify()
and self.forward_metadata.max_seq_len_q > 1
if self.forward_metadata.max_seq_len_q > 1
else None
)
o = self._run_fixed_q_len_decode(