[DSv4.1] Carry decoder tail counts through DP scheduler synchronization

On CUDA, take every rank's late-layer MoE gather size from the scheduled
per-rank trim counts, which ride the existing scheduler all-gather, and switch
and restore them with #43077's late-layer DP helpers; ROCm keeps its eager
count gather. Make the helpers' count tensor copy asynchronous so the switch
does not synchronize the stream. Restrict to attention TP 1 and TP MoE.

(cherry picked from commit a6c2d5f40efec4b491fcdd2dbcedf14ec70d3861)
This commit is contained in:
Yangmin Li
2026-11-04 09:45:07 +08:00
committed by minke.yu
parent 0b45c07637
commit 01087e689c
6 changed files with 265 additions and 9 deletions
@@ -2649,6 +2649,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# For DP attention
global_num_tokens: Optional[List[int]] = None
global_num_tokens_for_logprob: Optional[List[int]] = None
# Per DP rank, the rows the decoder SWA tail drops; None when no rank trims.
global_decoder_trim_rows: Optional[List[int]] = None
# The draft model can use a different MoE A2A backend than the target.
draft_global_num_tokens: Optional[List[int]] = None
draft_global_num_tokens_for_logprob: Optional[List[int]] = None
@@ -32,8 +32,15 @@ from sglang.srt.model_executor.cuda_graph_config import (
check_cuda_graph_backend,
cuda_graph_fully_disabled,
)
from sglang.srt.model_executor.encoder_swa_replay import folded_replay_rows
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.encoder_swa_replay import (
decoder_swa_trim_rows,
folded_replay_rows,
)
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
ForwardMode,
get_server_return_hidden_states_mode,
)
from sglang.srt.model_executor.runner import PrefillCudaGraphRunner
from sglang.srt.observability.metrics_collector import (
DPBalanceStats,
@@ -161,6 +168,8 @@ class MLPSyncBatchInfo:
is_health_check: bool = False
# Causal (query, key) pairs this rank attends this step.
attention_pairs: int = 0
# Rows the decoder SWA tail drops before the late layers.
decoder_trim_rows: int = 0
# some gathered elements
tp0_info_cpu: torch.Tensor = None
@@ -169,6 +178,7 @@ class MLPSyncBatchInfo:
global_num_tokens: list[int] = None
global_num_tokens_for_logprob: list[int] = None
global_attention_pairs: list[int] = None
global_decoder_trim_rows: list[int] = None
tbo_split_seq_index: torch.Tensor = None
global_forward_mode: int = None
dp_cooperation_info: Optional[DPCooperationInfo] = None
@@ -187,6 +197,7 @@ class MLPSyncBatchInfo:
int(self.can_run_draft_cuda_graph),
int(self.is_health_check),
self.attention_pairs,
self.decoder_trim_rows,
],
device=device,
dtype=dtype,
@@ -206,6 +217,7 @@ class MLPSyncBatchInfo:
1, # can_run_draft_cuda_graph
0, # is_health_check
0, # attention_pairs
0, # decoder_trim_rows
],
device=device,
dtype=dtype,
@@ -217,6 +229,7 @@ class MLPSyncBatchInfo:
self.global_num_tokens = [self.num_tokens]
self.global_num_tokens_for_logprob = [self.num_tokens_for_logprob]
self.global_attention_pairs = [self.attention_pairs]
self.global_decoder_trim_rows = [self.decoder_trim_rows]
self.any_health_check = self.is_health_check
if _ENABLE_METRICS_DP_ATTENTION:
self.dp_cooperation_info = DPCooperationInfo.create(
@@ -294,6 +307,7 @@ class MLPSyncBatchInfo:
self.can_run_draft_cuda_graph = bool(tp0_info_cpu[:, 8].min())
self.any_health_check = bool(tp0_info_cpu[:, 9].max())
self.global_attention_pairs = tp0_info_cpu[:, 10].tolist()
self.global_decoder_trim_rows = tp0_info_cpu[:, 11].tolist()
self.sync_wait_seconds = time.perf_counter() - sync_start
if _ENABLE_METRICS_DP_ATTENTION:
self.dp_cooperation_info = DPCooperationInfo.create(
@@ -308,6 +322,7 @@ def _update_gather_batch(
draft_require_mlp_tp_gather: Optional[bool] = None,
skip_global_metadata=False,
):
batch.global_decoder_trim_rows = None
if not require_mlp_tp_gather:
batch.global_num_tokens = [mlp_sync_info.num_tokens]
batch.global_num_tokens_for_logprob = [mlp_sync_info.num_tokens_for_logprob]
@@ -316,6 +331,12 @@ def _update_gather_batch(
batch.global_num_tokens_for_logprob = (
mlp_sync_info.global_num_tokens_for_logprob
)
if (
any(mlp_sync_info.global_decoder_trim_rows or ())
and min(mlp_sync_info.global_decoder_trim_rows) >= 0
):
# Every rank resizes its late-layer DP gather when any rank trims.
batch.global_decoder_trim_rows = mlp_sync_info.global_decoder_trim_rows
# Reuse the same all-gather result for a draft model whose A2A backend
# requires a different local/full token-count representation.
if draft_require_mlp_tp_gather is not None:
@@ -532,6 +553,7 @@ def prepare_mlp_sync_batch_raw(
attn_tp_size = parallel.attn_tp_size
tp_group = parallel.tp_group
# Check if other DP workers have running batches
decoder_trim_rows = 0
if (
local_batch is None
or local_batch.forward_mode.is_prebuilt()
@@ -549,6 +571,15 @@ def prepare_mlp_sync_batch_raw(
and model_runner.attn_backend.folds_encoder_swa_replay
):
num_tokens += folded_replay_rows(local_batch)
if get_exec().features.enable_decoder_swa_bounded_replay:
decoder_trim_rows = (
-1
if get_server_return_hidden_states_mode() == CaptureHiddenMode.FULL
else decoder_swa_trim_rows(
local_batch,
folds_encoder=model_runner.attn_backend.folds_encoder_swa_replay,
)
)
num_tokens_for_logprob = sum(
# We should have at least 1 token for sample in every case.
max(extend_len - logprob_start_len, 1)
@@ -641,6 +672,7 @@ def prepare_mlp_sync_batch_raw(
# Gathered unconditionally like the other columns, so a per-process
# metrics flag cannot make ranks disagree on what the tensor holds.
attention_pairs=_local_attention_pairs(local_batch),
decoder_trim_rows=decoder_trim_rows,
)
if num_dp_ranks == 1:
@@ -52,6 +52,23 @@ def folded_replay_rows(batch) -> int:
)
def decoder_swa_trim_rows(batch, *, folds_encoder: bool) -> int:
"""Rows removed by a plain extend's decoder tail; -1 requests the backend
exchange when prompt outputs require whole-extend rows."""
if not batch.forward_mode.is_extend_without_speculative():
return 0
if batch.return_logprob or batch.return_hidden_states:
return -1
lens = list(batch.extend_lens)
if folds_encoder and batch.encoder_swa_reset is not None:
for i, (reset, prefix) in enumerate(
zip(batch.encoder_swa_reset, batch.prefix_lens)
):
if reset:
lens[i] += replay_len(prefix)
return sum(max(0, n - 128) for n in lens)
def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None:
forward_batch.encoder_swa_row_floor = folded.row_floor
forward_batch.encoder_swa_compress_rows = folded.keep_rows
@@ -51,6 +51,7 @@ from sglang.srt.layers.dp_attention import (
set_is_extend_in_batch,
world_dp_gather_enabled,
)
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.model_executor.forward_batch_deepseek_mha_mixin import (
ForwardBatchDeepSeekMHAMixin,
)
@@ -59,6 +60,7 @@ from sglang.srt.runtime_context import (
get_flags,
get_lora,
get_parallel,
get_platform,
mamba_cache_chunk_size,
)
from sglang.srt.speculative.spec_info import SpecInputType
@@ -694,6 +696,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# Has to be None when cuda graph is captured.
global_num_tokens_for_logprob_cpu: Optional[List[int]] = None
global_num_tokens_for_logprob_gpu: Optional[torch.Tensor] = None
# Per DP rank, the rows the decoder SWA tail drops before the late layers;
# None when no rank trims (see DeepseekV4Model's late-layer DP switch).
global_decoder_trim_rows_cpu: Optional[List[int]] = None
# Real (non-padding) token count, held at two scopes whose meaning never
# changes once set:
@@ -1189,6 +1194,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
ret.init_mlp_sync_metadata(
batch, device, is_draft_worker=model_runner.is_draft_worker
)
if not model_runner.is_draft_worker:
ret.global_decoder_trim_rows_cpu = batch.global_decoder_trim_rows
if ret.forward_mode.is_idle():
ret.positions = torch.empty((0,), dtype=torch.int64, device=device)
@@ -1968,13 +1975,29 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
):
local_late_rows = model_runner.attn_backend.late_layer_num_tokens(self)
if local_late_rows is not None:
late_rows = torch.empty(sync_group_size, dtype=torch.int64)
all_gather_single(
late_rows,
torch.tensor([local_late_rows], dtype=torch.int64),
group=get_parallel().tp_group.cpu_group,
)
self.global_late_layer_num_tokens_cpu = late_rows.tolist()
trims = self.global_decoder_trim_rows_cpu
if (
trims is not None
and get_platform().is_cuda
and get_parallel().attn_tp_size == 1
and get_exec().graph.cuda_graph_config.prefill.backend
== Backend.DISABLED
and not self.dp_padding_mode.is_max_len()
):
counts = [
n - t for n, t in zip(global_num_tokens, trims, strict=True)
]
assert min(counts) >= 0, (global_num_tokens, trims)
assert counts[dp_slot_in(counts)] == local_late_rows
self.global_late_layer_num_tokens_cpu = counts
else:
late_rows = torch.empty(sync_group_size, dtype=torch.int64)
all_gather_single(
late_rows,
torch.tensor([local_late_rows], dtype=torch.int64),
group=get_parallel().tp_group.cpu_group,
)
self.global_late_layer_num_tokens_cpu = late_rows.tolist()
TboForwardBatchPreparer.prepare(
batch=self, is_draft_worker=model_runner.is_draft_worker