Support XQA backend for SpecDec verify (#32269)

This commit is contained in:
akhilg-nv
2026-09-24 20:14:46 -07:00
committed by GitHub
parent 6103a8c743
commit 37ebcac3c4
2 changed files with 160 additions and 0 deletions
@@ -396,6 +396,18 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
"SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS must be at least 1, "
f"got {self.decode_seq_len_splits}"
)
self._xqa_spec_dec_mask = None
if self.is_xqa_impl and self.speculative_num_draft_tokens:
draft_len = self.speculative_num_draft_tokens
max_bs = model_runner.max_running_requests + 1
words_per_row = (draft_len + 31) // 32 * 2
row = torch.arange(draft_len, dtype=torch.int32)[:, None]
word = torch.arange(words_per_row, dtype=torch.int32)[None, :]
bits = (row + 1 - 16 * word).clamp(0, 16)
mask = ((1 << bits) - 1).to(torch.uint16)
self._xqa_spec_dec_mask = (
mask.unsqueeze(0).expand(max_bs, -1, -1).contiguous().to(self.device)
)
def _check_decode_kv_access(self) -> None:
supported_kinds = {
@@ -1333,6 +1345,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
kv_cache_sf=None,
out: Optional[torch.Tensor] = None,
out_dtype: Optional[torch.dtype] = None,
mask: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Run decode, optionally sorting and splitting requests by KV length."""
@@ -1346,6 +1359,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
kwargs = {}
if q_len_per_req != 1:
kwargs["q_len_per_req"] = q_len_per_req
if mask is not None:
kwargs["mask"] = mask[: group_seq_lens.shape[0]]
return flashinfer.decode.trtllm_batch_decode_with_kv_cache(
query=group_query,
kv_cache=kv_cache,
@@ -1694,6 +1709,12 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
multi_ctas_kv_counter_buffer=self._multi_ctas_kv_counter_buffer,
)
else:
mask = (
self._xqa_spec_dec_mask
if forward_batch.forward_mode.is_target_verify()
and self.forward_metadata.max_seq_len_q > 1
else None
)
o = self._run_fixed_q_len_decode(
q,
kv_cache,
@@ -1707,6 +1728,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
out_dtype=(None if uses_native_fp4 else self.q_data_type),
kv_cache_sf=kv_cache_block_scales,
q_len_per_req=self.forward_metadata.max_seq_len_q,
mask=mask,
)
elif self.use_fmha_v2 and not cp_active:
# CP must go through cp_strategy.run_attention (per-shard