Support XQA backend for SpecDec verify (#32269)
This commit is contained in:
@@ -396,6 +396,18 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
"SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS must be at least 1, "
|
||||
f"got {self.decode_seq_len_splits}"
|
||||
)
|
||||
self._xqa_spec_dec_mask = None
|
||||
if self.is_xqa_impl and self.speculative_num_draft_tokens:
|
||||
draft_len = self.speculative_num_draft_tokens
|
||||
max_bs = model_runner.max_running_requests + 1
|
||||
words_per_row = (draft_len + 31) // 32 * 2
|
||||
row = torch.arange(draft_len, dtype=torch.int32)[:, None]
|
||||
word = torch.arange(words_per_row, dtype=torch.int32)[None, :]
|
||||
bits = (row + 1 - 16 * word).clamp(0, 16)
|
||||
mask = ((1 << bits) - 1).to(torch.uint16)
|
||||
self._xqa_spec_dec_mask = (
|
||||
mask.unsqueeze(0).expand(max_bs, -1, -1).contiguous().to(self.device)
|
||||
)
|
||||
|
||||
def _check_decode_kv_access(self) -> None:
|
||||
supported_kinds = {
|
||||
@@ -1333,6 +1345,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
kv_cache_sf=None,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
out_dtype: Optional[torch.dtype] = None,
|
||||
mask: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run decode, optionally sorting and splitting requests by KV length."""
|
||||
|
||||
@@ -1346,6 +1359,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
kwargs = {}
|
||||
if q_len_per_req != 1:
|
||||
kwargs["q_len_per_req"] = q_len_per_req
|
||||
if mask is not None:
|
||||
kwargs["mask"] = mask[: group_seq_lens.shape[0]]
|
||||
return flashinfer.decode.trtllm_batch_decode_with_kv_cache(
|
||||
query=group_query,
|
||||
kv_cache=kv_cache,
|
||||
@@ -1694,6 +1709,12 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
multi_ctas_kv_counter_buffer=self._multi_ctas_kv_counter_buffer,
|
||||
)
|
||||
else:
|
||||
mask = (
|
||||
self._xqa_spec_dec_mask
|
||||
if forward_batch.forward_mode.is_target_verify()
|
||||
and self.forward_metadata.max_seq_len_q > 1
|
||||
else None
|
||||
)
|
||||
o = self._run_fixed_q_len_decode(
|
||||
q,
|
||||
kv_cache,
|
||||
@@ -1707,6 +1728,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
out_dtype=(None if uses_native_fp4 else self.q_data_type),
|
||||
kv_cache_sf=kv_cache_block_scales,
|
||||
q_len_per_req=self.forward_metadata.max_seq_len_q,
|
||||
mask=mask,
|
||||
)
|
||||
elif self.use_fmha_v2 and not cp_active:
|
||||
# CP must go through cp_strategy.run_attention (per-shard
|
||||
|
||||
Reference in New Issue
Block a user