Fix XQA draft extend with FP8 KV cache in trtllm_mha (#41656)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
abd6c374c7
commit
55dcf3254f
@@ -1600,15 +1600,17 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
*self._kv_write_scales(layer),
|
||||
)
|
||||
|
||||
uses_decode_kernel = (
|
||||
forward_batch.forward_mode.is_target_verify()
|
||||
or forward_batch.forward_mode.is_draft_extend_v2()
|
||||
)
|
||||
|
||||
q_scale = 1.0
|
||||
if uses_native_fp4:
|
||||
q = q.to(torch.float8_e4m3fn)
|
||||
elif (
|
||||
self.data_type == torch.float8_e4m3fn
|
||||
and (
|
||||
not self.is_xqa_impl
|
||||
or not forward_batch.forward_mode.is_target_verify()
|
||||
)
|
||||
and (not self.is_xqa_impl or not uses_decode_kernel)
|
||||
and not use_fused_qkv
|
||||
):
|
||||
q = q.to(torch.float8_e4m3fn)
|
||||
@@ -1617,11 +1619,6 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
else:
|
||||
q = q.reshape(-1, layer.tp_q_head_num, layer.head_dim)
|
||||
|
||||
is_decode_mode = (
|
||||
forward_batch.forward_mode.is_target_verify()
|
||||
or forward_batch.forward_mode.is_draft_extend_v2()
|
||||
)
|
||||
|
||||
if uses_native_fp4:
|
||||
kv_cache, kv_cache_block_scales = self._get_nvfp4_decode_kv_cache(layer)
|
||||
k_cache, v_cache = kv_cache
|
||||
@@ -1631,7 +1628,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
k_cache_raw, v_cache_raw = self.token_to_kv_pool.get_kv_buffer(
|
||||
layer.layer_id
|
||||
)
|
||||
if not self.use_fmha_v2 or is_decode_mode:
|
||||
if not self.use_fmha_v2 or uses_decode_kernel:
|
||||
# Decode and SM100 batch_context kernels require HND layout.
|
||||
k_cache, v_cache = self._reshape_paged_kv_cache(
|
||||
k_cache_raw, v_cache_raw, layer, layer.head_dim
|
||||
@@ -1658,7 +1655,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
page_table = self._get_layer_page_table(layer, forward_batch)
|
||||
native_out = self._nvfp4_output_view(q) if uses_native_fp4 else None
|
||||
|
||||
if is_decode_mode:
|
||||
if uses_decode_kernel:
|
||||
if (
|
||||
forward_batch.forward_mode.is_target_verify()
|
||||
and layer.attn_type == AttentionType.ENCODER_ONLY
|
||||
@@ -1716,8 +1713,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
else:
|
||||
mask = (
|
||||
self._xqa_spec_dec_mask
|
||||
if forward_batch.forward_mode.is_target_verify()
|
||||
and self.forward_metadata.max_seq_len_q > 1
|
||||
if self.forward_metadata.max_seq_len_q > 1
|
||||
else None
|
||||
)
|
||||
o = self._run_fixed_q_len_decode(
|
||||
|
||||
Reference in New Issue
Block a user