[DP attention] Publish DP buffer sizes from a ForwardBatch (#40858)
Co-authored-by: hanminglu <hanminglu@fb.com> Co-authored-by: oss-sync bot (pranjalssh) <pranjalssh@users.noreply.github.com>
This commit is contained in:
co-authored by
hanminglu
oss-sync bot
parent
86b3558b41
commit
f404db94d4
@@ -823,6 +823,9 @@ class TboForwardBatchPreparer:
|
||||
_original_num_tokens=None,
|
||||
global_num_tokens_gpu=None,
|
||||
global_num_tokens_cpu=None,
|
||||
# Children publish no per-rank list of their own; the parent
|
||||
# published the gather sizes before it was split.
|
||||
global_num_tokens_padded_cpu=None,
|
||||
global_dp_buffer_len=global_dp_buffer_len,
|
||||
global_num_tokens_for_logprob_gpu=None,
|
||||
global_num_tokens_for_logprob_cpu=None,
|
||||
|
||||
@@ -337,6 +337,24 @@ def set_local_dp_buffer_len(local_dp_buffer_len: int) -> None:
|
||||
_DpGatheredBufferWrapper.set_local_dp_buffer_len(local_dp_buffer_len)
|
||||
|
||||
|
||||
def set_dp_buffer_len_from_batch(forward_batch: ForwardBatch) -> None:
|
||||
"""Publish the DP gather sizes ``forward_batch`` carries: the buffer
|
||||
length, the per-rank token counts as padded for the gather, this rank's
|
||||
entry, and the padding mode. Capture batches carry no separately padded
|
||||
list, so the raw counts stand in for it."""
|
||||
global_num_tokens = forward_batch.global_num_tokens_padded_cpu
|
||||
if global_num_tokens is None:
|
||||
global_num_tokens = forward_batch.global_num_tokens_cpu
|
||||
dp_rank = get_parallel().attn_dp_rank if len(global_num_tokens) > 1 else 0
|
||||
set_dp_buffer_len(
|
||||
forward_batch.global_dp_buffer_len,
|
||||
global_num_tokens[dp_rank],
|
||||
forward_batch.dp_padding_mode.is_max_len(),
|
||||
global_num_tokens,
|
||||
forward_batch.global_num_tokens_gpu,
|
||||
)
|
||||
|
||||
|
||||
def get_dp_global_num_tokens() -> List[int]:
|
||||
return _DpGatheredBufferWrapper.get_dp_global_num_tokens()
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ from sglang.srt.kv_canary.req_to_expected_token_ids_manager import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
DpPaddingMode,
|
||||
dp_gather_slot,
|
||||
set_dp_buffer_len,
|
||||
set_dp_buffer_len_from_batch,
|
||||
set_is_extend_in_batch,
|
||||
world_dp_gather_enabled,
|
||||
)
|
||||
@@ -688,6 +688,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
dp_local_start_pos: Optional[torch.Tensor] = None # cached info at runtime
|
||||
dp_local_num_tokens: Optional[torch.Tensor] = None # cached info at runtime
|
||||
global_dp_buffer_len: Optional[int] = None
|
||||
# global_num_tokens_cpu as published for the DP gather: attn-TP aligned and,
|
||||
# under MAX_LEN, padded to the max. None when the raw list already is.
|
||||
global_num_tokens_padded_cpu: Optional[List[int]] = None
|
||||
|
||||
# For Qwen2-VL
|
||||
mrope_positions: torch.Tensor = None
|
||||
@@ -1584,13 +1587,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
)
|
||||
|
||||
self.global_dp_buffer_len = buffer_len
|
||||
set_dp_buffer_len(
|
||||
buffer_len,
|
||||
num_tokens,
|
||||
dp_padding_mode.is_max_len(),
|
||||
global_num_tokens,
|
||||
self.global_num_tokens_gpu,
|
||||
)
|
||||
self.global_num_tokens_padded_cpu = global_num_tokens
|
||||
set_dp_buffer_len_from_batch(self)
|
||||
set_is_extend_in_batch(self.is_extend_in_batch)
|
||||
|
||||
bs = self.batch_size
|
||||
|
||||
Reference in New Issue
Block a user