perf(triton): avoid per-step D2H .item() sync in cuda-graph loc translate (#29921)

Co-authored-by: lch1475369 <lch1475369@gmail.com>
This commit is contained in:
Cheng Wan
2026-07-02 13:20:00 -07:00
committed by GitHub
co-authored by lch1475369
parent 85e71b7e13
commit bc25abb786
@@ -636,15 +636,31 @@ class TritonAttnBackend(AttentionBackend):
"""
if self._translate_kv_loc is None:
return None
# Full-attention read path.
n_kv = int(self.kv_indptr[bs].item())
# seq_lens_sum is the reliable "mirror present" signal: it is
# None-preserving into the replay view, unlike seq_lens_cpu (always a
# non-None but stale slice for gpu_only batches). None -> fall back to a
# per-step D2H `.item()` on the indptr.
have_cpu_mirror = forward_batch.seq_lens_sum is not None
# Full-attention read path. kv_indptr[bs] == seq_lens_sum.
n_kv = (
forward_batch.seq_lens_sum
if have_cpu_mirror
else int(self.kv_indptr[bs].item())
)
if n_kv > 0:
self.cuda_graph_kv_indices[:n_kv] = self._translate_kv_loc(
self.cuda_graph_kv_indices[:n_kv]
)
# SWA window read path (hybrid-SWA unified pools only).
# SWA window read path. window_kv_indptr[bs] == sum(min(seq_len, window)).
if self.sliding_window_size is not None and self.sliding_window_size > 0:
n_win = int(self.window_kv_indptr[bs].item())
if have_cpu_mirror:
n_win = int(
forward_batch.seq_lens_cpu[:bs]
.clamp(max=self.sliding_window_size)
.sum()
)
else:
n_win = int(self.window_kv_indptr[bs].item())
if n_win > 0:
self.cuda_graph_window_kv_indices[:n_win] = (
self.token_to_kv_pool.translate_loc_from_full_to_swa(