[Fix] kda: Keep FlashInfer prefill off its grouped kernel's bf16/flat-dt_bias contract (#45018)

This commit is contained in:
Rain Jiang
2026-10-19 13:16:54 -07:00
committed by GitHub
parent f1a2ee763d
commit aaa9388c91
3 changed files with 191 additions and 6 deletions
@@ -402,10 +402,16 @@ class KDAKernelDispatcher:
**kwargs,
)
def effective_extend_kernel(self, lower_bound: Optional[float], num_tokens: int):
"""The packed FlashInfer prefill kernel requires more than one token."""
def effective_extend_kernel(
self, lower_bound: Optional[float], num_tokens: int, num_seqs: int = 1
):
"""The packed FlashInfer prefill kernel requires more than one token per
sequence. A batch where every sequence extends exactly one token (e.g.
retracted requests re-entering with a fully cached prefix) would fall to
FlashInfer's grouped recurrent kernel, which only accepts a bf16 state
pool; the KDA pool is fp32, so route those batches to Triton."""
kernel = self.extend_kernel
if self.prefill_backend.is_flashinfer() and num_tokens <= 1:
if self.prefill_backend.is_flashinfer() and num_tokens <= max(num_seqs, 1):
return self.triton_kernel
if lower_bound is not None and not getattr(kernel, "supports_safe_gate", True):
kernel = self.triton_kernel
@@ -424,7 +430,9 @@ class KDAKernelDispatcher:
query_start_loc: torch.Tensor,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor | None]:
kernel = self.effective_extend_kernel(kwargs.get("lower_bound"), q.shape[1])
kernel = self.effective_extend_kernel(
kwargs.get("lower_bound"), q.shape[1], query_start_loc.numel() - 1
)
return kernel.extend(
q,
k,
@@ -1110,7 +1118,7 @@ class KDAAttnBackend(MambaAttnBackendBase):
# Check the kernel the dispatcher will actually run (safe-gate
# reroute included), not just the configured one.
extend_kernel = self.kernel_dispatcher.effective_extend_kernel(
layer.lower_bound, q.shape[1]
layer.lower_bound, q.shape[1], query_start_loc.numel() - 1
)
assert extend_kernel.supports_track_state_snapshot, (
f"{type(extend_kernel).__name__} cannot write the fp32 track "
@@ -121,7 +121,9 @@ class FlashInferKDAPrefillKernel(LinearAttnKernelBase):
g=g[:, : q.shape[1]].contiguous(),
beta=beta[:, : q.shape[1]].contiguous(),
A_log=A_log.reshape(-1).float().contiguous(),
dt_bias=dt_bias.reshape(q.shape[2], 128).float().contiguous(),
# FlashInfer's grouped-CTA recurrent kernel (extends with a cached prefix)
# indexes dt_bias as a flat [H*K] tensor and asserts a unit leading stride.
dt_bias=dt_bias.reshape(-1).float().contiguous(),
initial_state=ssm_states,
output_final_state=False,
use_qk_l2norm_in_kernel=True,