[Fix] Size the prefill delayer's gather buffer to its TP group (#41809)

This commit is contained in:
Cheng Wan
2026-09-30 13:26:12 -07:00
committed by GitHub
parent ecc0f0dec2
commit a659e6461f
2 changed files with 71 additions and 2 deletions
@@ -129,9 +129,10 @@ class PrefillDelayer:
# Fields packed per rank into the all-gather tensor: prefillable,
# token_watermark_force_allow, running_batch, max_prefill_bs,
# waiting_queue_len.
# waiting_queue_len. The gather spans the TP group, whose ranks are
# laid out DP-major over attn_cp_size * attn_tp_size.
self._global_info_buffer = torch.empty(
(dp_size_dim, parallel.attn_tp_size, 5),
(dp_size_dim, parallel.attn_cp_size * parallel.attn_tp_size, 5),
dtype=torch.int64,
device=self._gather_device,
)