Fix XQA draft extend with FP8 KV cache in trtllm_mha (#41656)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
abd6c374c7
commit
55dcf3254f
@@ -1600,15 +1600,17 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
*self._kv_write_scales(layer),
|
||||
)
|
||||
|
||||
uses_decode_kernel = (
|
||||
forward_batch.forward_mode.is_target_verify()
|
||||
or forward_batch.forward_mode.is_draft_extend_v2()
|
||||
)
|
||||
|
||||
q_scale = 1.0
|
||||
if uses_native_fp4:
|
||||
q = q.to(torch.float8_e4m3fn)
|
||||
elif (
|
||||
self.data_type == torch.float8_e4m3fn
|
||||
and (
|
||||
not self.is_xqa_impl
|
||||
or not forward_batch.forward_mode.is_target_verify()
|
||||
)
|
||||
and (not self.is_xqa_impl or not uses_decode_kernel)
|
||||
and not use_fused_qkv
|
||||
):
|
||||
q = q.to(torch.float8_e4m3fn)
|
||||
@@ -1617,11 +1619,6 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
else:
|
||||
q = q.reshape(-1, layer.tp_q_head_num, layer.head_dim)
|
||||
|
||||
is_decode_mode = (
|
||||
forward_batch.forward_mode.is_target_verify()
|
||||
or forward_batch.forward_mode.is_draft_extend_v2()
|
||||
)
|
||||
|
||||
if uses_native_fp4:
|
||||
kv_cache, kv_cache_block_scales = self._get_nvfp4_decode_kv_cache(layer)
|
||||
k_cache, v_cache = kv_cache
|
||||
@@ -1631,7 +1628,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
k_cache_raw, v_cache_raw = self.token_to_kv_pool.get_kv_buffer(
|
||||
layer.layer_id
|
||||
)
|
||||
if not self.use_fmha_v2 or is_decode_mode:
|
||||
if not self.use_fmha_v2 or uses_decode_kernel:
|
||||
# Decode and SM100 batch_context kernels require HND layout.
|
||||
k_cache, v_cache = self._reshape_paged_kv_cache(
|
||||
k_cache_raw, v_cache_raw, layer, layer.head_dim
|
||||
@@ -1658,7 +1655,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
page_table = self._get_layer_page_table(layer, forward_batch)
|
||||
native_out = self._nvfp4_output_view(q) if uses_native_fp4 else None
|
||||
|
||||
if is_decode_mode:
|
||||
if uses_decode_kernel:
|
||||
if (
|
||||
forward_batch.forward_mode.is_target_verify()
|
||||
and layer.attn_type == AttentionType.ENCODER_ONLY
|
||||
@@ -1716,8 +1713,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
else:
|
||||
mask = (
|
||||
self._xqa_spec_dec_mask
|
||||
if forward_batch.forward_mode.is_target_verify()
|
||||
and self.forward_metadata.max_seq_len_q > 1
|
||||
if self.forward_metadata.max_seq_len_q > 1
|
||||
else None
|
||||
)
|
||||
o = self._run_fixed_q_len_decode(
|
||||
|
||||
@@ -101,21 +101,7 @@ def test_xqa_mask_not_allocated_when_unused(make_xqa_backend, draft_len, is_xqa)
|
||||
assert make_xqa_backend(draft_len, is_xqa)._xqa_spec_dec_mask is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("splits", [1, 2, 8])
|
||||
@pytest.mark.parametrize(
|
||||
"is_xqa,mode,width,masked",
|
||||
[
|
||||
(True, ForwardMode.TARGET_VERIFY, 8, True),
|
||||
(False, ForwardMode.TARGET_VERIFY, 8, False),
|
||||
(True, ForwardMode.TARGET_VERIFY, 1, False),
|
||||
(True, ForwardMode.DRAFT_EXTEND_V2, 1, False),
|
||||
],
|
||||
)
|
||||
def test_xqa_verify_mask_forwarding(
|
||||
make_xqa_backend, monkeypatch, splits, is_xqa, mode, width, masked
|
||||
):
|
||||
backend = make_xqa_backend(width, is_xqa)
|
||||
backend.decode_seq_len_splits = splits
|
||||
def _run_decode_mode_extend(backend, monkeypatch, *, mode, width, decode):
|
||||
bs = 5
|
||||
seq_lens = torch.tensor([30, 10, 50, 20, 40], dtype=torch.int32)
|
||||
page_table = torch.arange(bs, dtype=torch.int32)[:, None]
|
||||
@@ -128,19 +114,6 @@ def test_xqa_verify_mask_forwarding(
|
||||
backend._get_bmm_scales = lambda *a: (1.0, 1.0)
|
||||
backend._get_layer_page_table = lambda *a: page_table
|
||||
monkeypatch.setattr(trtllm_mha_backend, "is_cp_active", lambda _: False)
|
||||
calls = []
|
||||
|
||||
def decode(**kwargs):
|
||||
calls.append(kwargs)
|
||||
group_bs = kwargs["seq_lens"].numel()
|
||||
mask = kwargs.get("mask")
|
||||
if masked:
|
||||
assert mask.shape == (group_bs, width, 2)
|
||||
assert mask.data_ptr() == backend._xqa_spec_dec_mask.data_ptr()
|
||||
else:
|
||||
assert mask is None
|
||||
return kwargs["query"] + 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
trtllm_mha_backend,
|
||||
"flashinfer",
|
||||
@@ -163,8 +136,70 @@ def test_xqa_verify_mask_forwarding(
|
||||
)
|
||||
fb = SimpleNamespace(out_cache_loc=None, forward_mode=mode, batch_size=bs)
|
||||
out = backend.forward_extend(q, None, None, layer, fb, save_kv_cache=False)
|
||||
return q, out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("splits", [1, 2, 8])
|
||||
@pytest.mark.parametrize(
|
||||
"is_xqa,mode,width,masked",
|
||||
[
|
||||
(True, ForwardMode.TARGET_VERIFY, 8, True),
|
||||
(False, ForwardMode.TARGET_VERIFY, 8, False),
|
||||
(True, ForwardMode.TARGET_VERIFY, 1, False),
|
||||
(True, ForwardMode.DRAFT_EXTEND_V2, 1, False),
|
||||
(True, ForwardMode.DRAFT_EXTEND_V2, 8, True),
|
||||
],
|
||||
)
|
||||
def test_xqa_verify_mask_forwarding(
|
||||
make_xqa_backend, monkeypatch, splits, is_xqa, mode, width, masked
|
||||
):
|
||||
"""XQA needs the draft-block mask whenever q_len > 1, draft extend included."""
|
||||
backend = make_xqa_backend(width, is_xqa)
|
||||
backend.decode_seq_len_splits = splits
|
||||
calls = []
|
||||
|
||||
def decode(**kwargs):
|
||||
calls.append(kwargs)
|
||||
group_bs = kwargs["seq_lens"].numel()
|
||||
mask = kwargs.get("mask")
|
||||
if masked:
|
||||
assert mask.shape == (group_bs, width, 2)
|
||||
assert mask.data_ptr() == backend._xqa_spec_dec_mask.data_ptr()
|
||||
else:
|
||||
assert mask is None
|
||||
return kwargs["query"] + 1
|
||||
|
||||
q, out = _run_decode_mode_extend(
|
||||
backend, monkeypatch, mode=mode, width=width, decode=decode
|
||||
)
|
||||
torch.testing.assert_close(out, (q + 1).view(-1, 8))
|
||||
assert len(calls) == min(splits, bs)
|
||||
assert len(calls) == min(splits, q.shape[0] // width)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"is_xqa,mode,expected_q_dtype",
|
||||
[
|
||||
(True, ForwardMode.TARGET_VERIFY, torch.bfloat16),
|
||||
(True, ForwardMode.DRAFT_EXTEND_V2, torch.bfloat16),
|
||||
(False, ForwardMode.DRAFT_EXTEND_V2, torch.float8_e4m3fn),
|
||||
],
|
||||
)
|
||||
def test_decode_mode_query_dtype_with_fp8_kv(
|
||||
make_xqa_backend, monkeypatch, is_xqa, mode, expected_q_dtype
|
||||
):
|
||||
"""With FP8 KV, XQA verify and draft extend keep Q bf16; trtllm-gen casts to FP8."""
|
||||
width = 8
|
||||
backend = make_xqa_backend(width, is_xqa)
|
||||
backend.data_type = torch.float8_e4m3fn
|
||||
backend.decode_seq_len_splits = 1
|
||||
query_dtypes = []
|
||||
|
||||
def decode(**kwargs):
|
||||
query_dtypes.append(kwargs["query"].dtype)
|
||||
return torch.zeros(kwargs["query"].shape, dtype=kwargs["out_dtype"])
|
||||
|
||||
_run_decode_mode_extend(backend, monkeypatch, mode=mode, width=width, decode=decode)
|
||||
assert query_dtypes == [expected_q_dtype]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
Reference in New Issue
Block a user