[DP attention] Publish DP buffer sizes from a ForwardBatch (#40858)

Co-authored-by: hanminglu <hanminglu@fb.com>
Co-authored-by: oss-sync bot (pranjalssh) <pranjalssh@users.noreply.github.com>
This commit is contained in:
Hanming Lu
2026-09-24 11:34:19 -07:00
committed by GitHub
co-authored by hanminglu oss-sync bot
parent 86b3558b41
commit f404db94d4
6 changed files with 109 additions and 9 deletions
@@ -823,6 +823,9 @@ class TboForwardBatchPreparer:
_original_num_tokens=None,
global_num_tokens_gpu=None,
global_num_tokens_cpu=None,
# Children publish no per-rank list of their own; the parent
# published the gather sizes before it was split.
global_num_tokens_padded_cpu=None,
global_dp_buffer_len=global_dp_buffer_len,
global_num_tokens_for_logprob_gpu=None,
global_num_tokens_for_logprob_cpu=None,
+18
View File
@@ -337,6 +337,24 @@ def set_local_dp_buffer_len(local_dp_buffer_len: int) -> None:
_DpGatheredBufferWrapper.set_local_dp_buffer_len(local_dp_buffer_len)
def set_dp_buffer_len_from_batch(forward_batch: ForwardBatch) -> None:
"""Publish the DP gather sizes ``forward_batch`` carries: the buffer
length, the per-rank token counts as padded for the gather, this rank's
entry, and the padding mode. Capture batches carry no separately padded
list, so the raw counts stand in for it."""
global_num_tokens = forward_batch.global_num_tokens_padded_cpu
if global_num_tokens is None:
global_num_tokens = forward_batch.global_num_tokens_cpu
dp_rank = get_parallel().attn_dp_rank if len(global_num_tokens) > 1 else 0
set_dp_buffer_len(
forward_batch.global_dp_buffer_len,
global_num_tokens[dp_rank],
forward_batch.dp_padding_mode.is_max_len(),
global_num_tokens,
forward_batch.global_num_tokens_gpu,
)
def get_dp_global_num_tokens() -> List[int]:
return _DpGatheredBufferWrapper.get_dp_global_num_tokens()
@@ -46,7 +46,7 @@ from sglang.srt.kv_canary.req_to_expected_token_ids_manager import (
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
dp_gather_slot,
set_dp_buffer_len,
set_dp_buffer_len_from_batch,
set_is_extend_in_batch,
world_dp_gather_enabled,
)
@@ -688,6 +688,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
dp_local_start_pos: Optional[torch.Tensor] = None # cached info at runtime
dp_local_num_tokens: Optional[torch.Tensor] = None # cached info at runtime
global_dp_buffer_len: Optional[int] = None
# global_num_tokens_cpu as published for the DP gather: attn-TP aligned and,
# under MAX_LEN, padded to the max. None when the raw list already is.
global_num_tokens_padded_cpu: Optional[List[int]] = None
# For Qwen2-VL
mrope_positions: torch.Tensor = None
@@ -1584,13 +1587,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
)
self.global_dp_buffer_len = buffer_len
set_dp_buffer_len(
buffer_len,
num_tokens,
dp_padding_mode.is_max_len(),
global_num_tokens,
self.global_num_tokens_gpu,
)
self.global_num_tokens_padded_cpu = global_num_tokens
set_dp_buffer_len_from_batch(self)
set_is_extend_in_batch(self.is_extend_in_batch)
bs = self.batch_size