Fix XQA draft extend with FP8 KV cache in trtllm_mha (#41656)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
elvischenv
2026-10-05 01:37:54 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent abd6c374c7
commit 55dcf3254f
2 changed files with 73 additions and 42 deletions
@@ -1600,15 +1600,17 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
*self._kv_write_scales(layer),
)
uses_decode_kernel = (
forward_batch.forward_mode.is_target_verify()
or forward_batch.forward_mode.is_draft_extend_v2()
)
q_scale = 1.0
if uses_native_fp4:
q = q.to(torch.float8_e4m3fn)
elif (
self.data_type == torch.float8_e4m3fn
and (
not self.is_xqa_impl
or not forward_batch.forward_mode.is_target_verify()
)
and (not self.is_xqa_impl or not uses_decode_kernel)
and not use_fused_qkv
):
q = q.to(torch.float8_e4m3fn)
@@ -1617,11 +1619,6 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
else:
q = q.reshape(-1, layer.tp_q_head_num, layer.head_dim)
is_decode_mode = (
forward_batch.forward_mode.is_target_verify()
or forward_batch.forward_mode.is_draft_extend_v2()
)
if uses_native_fp4:
kv_cache, kv_cache_block_scales = self._get_nvfp4_decode_kv_cache(layer)
k_cache, v_cache = kv_cache
@@ -1631,7 +1628,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
k_cache_raw, v_cache_raw = self.token_to_kv_pool.get_kv_buffer(
layer.layer_id
)
if not self.use_fmha_v2 or is_decode_mode:
if not self.use_fmha_v2 or uses_decode_kernel:
# Decode and SM100 batch_context kernels require HND layout.
k_cache, v_cache = self._reshape_paged_kv_cache(
k_cache_raw, v_cache_raw, layer, layer.head_dim
@@ -1658,7 +1655,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
page_table = self._get_layer_page_table(layer, forward_batch)
native_out = self._nvfp4_output_view(q) if uses_native_fp4 else None
if is_decode_mode:
if uses_decode_kernel:
if (
forward_batch.forward_mode.is_target_verify()
and layer.attn_type == AttentionType.ENCODER_ONLY
@@ -1716,8 +1713,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
else:
mask = (
self._xqa_spec_dec_mask
if forward_batch.forward_mode.is_target_verify()
and self.forward_metadata.max_seq_len_q > 1
if self.forward_metadata.max_seq_len_q > 1
else None
)
o = self._run_fixed_q_len_decode(
@@ -101,21 +101,7 @@ def test_xqa_mask_not_allocated_when_unused(make_xqa_backend, draft_len, is_xqa)
assert make_xqa_backend(draft_len, is_xqa)._xqa_spec_dec_mask is None
@pytest.mark.parametrize("splits", [1, 2, 8])
@pytest.mark.parametrize(
"is_xqa,mode,width,masked",
[
(True, ForwardMode.TARGET_VERIFY, 8, True),
(False, ForwardMode.TARGET_VERIFY, 8, False),
(True, ForwardMode.TARGET_VERIFY, 1, False),
(True, ForwardMode.DRAFT_EXTEND_V2, 1, False),
],
)
def test_xqa_verify_mask_forwarding(
make_xqa_backend, monkeypatch, splits, is_xqa, mode, width, masked
):
backend = make_xqa_backend(width, is_xqa)
backend.decode_seq_len_splits = splits
def _run_decode_mode_extend(backend, monkeypatch, *, mode, width, decode):
bs = 5
seq_lens = torch.tensor([30, 10, 50, 20, 40], dtype=torch.int32)
page_table = torch.arange(bs, dtype=torch.int32)[:, None]
@@ -128,19 +114,6 @@ def test_xqa_verify_mask_forwarding(
backend._get_bmm_scales = lambda *a: (1.0, 1.0)
backend._get_layer_page_table = lambda *a: page_table
monkeypatch.setattr(trtllm_mha_backend, "is_cp_active", lambda _: False)
calls = []
def decode(**kwargs):
calls.append(kwargs)
group_bs = kwargs["seq_lens"].numel()
mask = kwargs.get("mask")
if masked:
assert mask.shape == (group_bs, width, 2)
assert mask.data_ptr() == backend._xqa_spec_dec_mask.data_ptr()
else:
assert mask is None
return kwargs["query"] + 1
monkeypatch.setattr(
trtllm_mha_backend,
"flashinfer",
@@ -163,8 +136,70 @@ def test_xqa_verify_mask_forwarding(
)
fb = SimpleNamespace(out_cache_loc=None, forward_mode=mode, batch_size=bs)
out = backend.forward_extend(q, None, None, layer, fb, save_kv_cache=False)
return q, out
@pytest.mark.parametrize("splits", [1, 2, 8])
@pytest.mark.parametrize(
"is_xqa,mode,width,masked",
[
(True, ForwardMode.TARGET_VERIFY, 8, True),
(False, ForwardMode.TARGET_VERIFY, 8, False),
(True, ForwardMode.TARGET_VERIFY, 1, False),
(True, ForwardMode.DRAFT_EXTEND_V2, 1, False),
(True, ForwardMode.DRAFT_EXTEND_V2, 8, True),
],
)
def test_xqa_verify_mask_forwarding(
make_xqa_backend, monkeypatch, splits, is_xqa, mode, width, masked
):
"""XQA needs the draft-block mask whenever q_len > 1, draft extend included."""
backend = make_xqa_backend(width, is_xqa)
backend.decode_seq_len_splits = splits
calls = []
def decode(**kwargs):
calls.append(kwargs)
group_bs = kwargs["seq_lens"].numel()
mask = kwargs.get("mask")
if masked:
assert mask.shape == (group_bs, width, 2)
assert mask.data_ptr() == backend._xqa_spec_dec_mask.data_ptr()
else:
assert mask is None
return kwargs["query"] + 1
q, out = _run_decode_mode_extend(
backend, monkeypatch, mode=mode, width=width, decode=decode
)
torch.testing.assert_close(out, (q + 1).view(-1, 8))
assert len(calls) == min(splits, bs)
assert len(calls) == min(splits, q.shape[0] // width)
@pytest.mark.parametrize(
"is_xqa,mode,expected_q_dtype",
[
(True, ForwardMode.TARGET_VERIFY, torch.bfloat16),
(True, ForwardMode.DRAFT_EXTEND_V2, torch.bfloat16),
(False, ForwardMode.DRAFT_EXTEND_V2, torch.float8_e4m3fn),
],
)
def test_decode_mode_query_dtype_with_fp8_kv(
make_xqa_backend, monkeypatch, is_xqa, mode, expected_q_dtype
):
"""With FP8 KV, XQA verify and draft extend keep Q bf16; trtllm-gen casts to FP8."""
width = 8
backend = make_xqa_backend(width, is_xqa)
backend.data_type = torch.float8_e4m3fn
backend.decode_seq_len_splits = 1
query_dtypes = []
def decode(**kwargs):
query_dtypes.append(kwargs["query"].dtype)
return torch.zeros(kwargs["query"].shape, dtype=kwargs["out_dtype"])
_run_decode_mode_extend(backend, monkeypatch, mode=mode, width=width, decode=decode)
assert query_dtypes == [expected_q_dtype]
@pytest.mark.parametrize(