From 01087e689cce8108ec806c00ce8c0b498ad4787f Mon Sep 17 00:00:00 2001 From: Yangmin Li Date: Fri, 9 Oct 2026 23:25:00 -0700 Subject: [PATCH] [DSv4.1] Carry decoder tail counts through DP scheduler synchronization On CUDA, take every rank's late-layer MoE gather size from the scheduled per-rank trim counts, which ride the existing scheduler all-gather, and switch and restore them with #43077's late-layer DP helpers; ROCm keeps its eager count gather. Make the helpers' count tensor copy asynchronous so the switch does not synchronize the stream. Restrict to attention TP 1 and TP MoE. (cherry picked from commit a6c2d5f40efec4b491fcdd2dbcedf14ec70d3861) --- python/sglang/srt/managers/schedule_batch.py | 2 + .../managers/scheduler_components/dp_attn.py | 36 +++- .../srt/model_executor/encoder_swa_replay.py | 17 ++ .../srt/model_executor/forward_batch_info.py | 37 +++- .../test_decoder_swa_scheduler_counts.py | 181 ++++++++++++++++++ .../spec/test_dp_spec_prefill_coordination.py | 1 + 6 files changed, 265 insertions(+), 9 deletions(-) create mode 100644 test/registered/unit/model_executor/test_decoder_swa_scheduler_counts.py diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 6cab5b8066..ee851774ff 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -2649,6 +2649,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # For DP attention global_num_tokens: Optional[List[int]] = None global_num_tokens_for_logprob: Optional[List[int]] = None + # Per DP rank, the rows the decoder SWA tail drops; None when no rank trims. + global_decoder_trim_rows: Optional[List[int]] = None # The draft model can use a different MoE A2A backend than the target. draft_global_num_tokens: Optional[List[int]] = None draft_global_num_tokens_for_logprob: Optional[List[int]] = None diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index bd72eaf906..e485f1ca6c 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -32,8 +32,15 @@ from sglang.srt.model_executor.cuda_graph_config import ( check_cuda_graph_backend, cuda_graph_fully_disabled, ) -from sglang.srt.model_executor.encoder_swa_replay import folded_replay_rows -from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.model_executor.encoder_swa_replay import ( + decoder_swa_trim_rows, + folded_replay_rows, +) +from sglang.srt.model_executor.forward_batch_info import ( + CaptureHiddenMode, + ForwardMode, + get_server_return_hidden_states_mode, +) from sglang.srt.model_executor.runner import PrefillCudaGraphRunner from sglang.srt.observability.metrics_collector import ( DPBalanceStats, @@ -161,6 +168,8 @@ class MLPSyncBatchInfo: is_health_check: bool = False # Causal (query, key) pairs this rank attends this step. attention_pairs: int = 0 + # Rows the decoder SWA tail drops before the late layers. + decoder_trim_rows: int = 0 # some gathered elements tp0_info_cpu: torch.Tensor = None @@ -169,6 +178,7 @@ class MLPSyncBatchInfo: global_num_tokens: list[int] = None global_num_tokens_for_logprob: list[int] = None global_attention_pairs: list[int] = None + global_decoder_trim_rows: list[int] = None tbo_split_seq_index: torch.Tensor = None global_forward_mode: int = None dp_cooperation_info: Optional[DPCooperationInfo] = None @@ -187,6 +197,7 @@ class MLPSyncBatchInfo: int(self.can_run_draft_cuda_graph), int(self.is_health_check), self.attention_pairs, + self.decoder_trim_rows, ], device=device, dtype=dtype, @@ -206,6 +217,7 @@ class MLPSyncBatchInfo: 1, # can_run_draft_cuda_graph 0, # is_health_check 0, # attention_pairs + 0, # decoder_trim_rows ], device=device, dtype=dtype, @@ -217,6 +229,7 @@ class MLPSyncBatchInfo: self.global_num_tokens = [self.num_tokens] self.global_num_tokens_for_logprob = [self.num_tokens_for_logprob] self.global_attention_pairs = [self.attention_pairs] + self.global_decoder_trim_rows = [self.decoder_trim_rows] self.any_health_check = self.is_health_check if _ENABLE_METRICS_DP_ATTENTION: self.dp_cooperation_info = DPCooperationInfo.create( @@ -294,6 +307,7 @@ class MLPSyncBatchInfo: self.can_run_draft_cuda_graph = bool(tp0_info_cpu[:, 8].min()) self.any_health_check = bool(tp0_info_cpu[:, 9].max()) self.global_attention_pairs = tp0_info_cpu[:, 10].tolist() + self.global_decoder_trim_rows = tp0_info_cpu[:, 11].tolist() self.sync_wait_seconds = time.perf_counter() - sync_start if _ENABLE_METRICS_DP_ATTENTION: self.dp_cooperation_info = DPCooperationInfo.create( @@ -308,6 +322,7 @@ def _update_gather_batch( draft_require_mlp_tp_gather: Optional[bool] = None, skip_global_metadata=False, ): + batch.global_decoder_trim_rows = None if not require_mlp_tp_gather: batch.global_num_tokens = [mlp_sync_info.num_tokens] batch.global_num_tokens_for_logprob = [mlp_sync_info.num_tokens_for_logprob] @@ -316,6 +331,12 @@ def _update_gather_batch( batch.global_num_tokens_for_logprob = ( mlp_sync_info.global_num_tokens_for_logprob ) + if ( + any(mlp_sync_info.global_decoder_trim_rows or ()) + and min(mlp_sync_info.global_decoder_trim_rows) >= 0 + ): + # Every rank resizes its late-layer DP gather when any rank trims. + batch.global_decoder_trim_rows = mlp_sync_info.global_decoder_trim_rows # Reuse the same all-gather result for a draft model whose A2A backend # requires a different local/full token-count representation. if draft_require_mlp_tp_gather is not None: @@ -532,6 +553,7 @@ def prepare_mlp_sync_batch_raw( attn_tp_size = parallel.attn_tp_size tp_group = parallel.tp_group # Check if other DP workers have running batches + decoder_trim_rows = 0 if ( local_batch is None or local_batch.forward_mode.is_prebuilt() @@ -549,6 +571,15 @@ def prepare_mlp_sync_batch_raw( and model_runner.attn_backend.folds_encoder_swa_replay ): num_tokens += folded_replay_rows(local_batch) + if get_exec().features.enable_decoder_swa_bounded_replay: + decoder_trim_rows = ( + -1 + if get_server_return_hidden_states_mode() == CaptureHiddenMode.FULL + else decoder_swa_trim_rows( + local_batch, + folds_encoder=model_runner.attn_backend.folds_encoder_swa_replay, + ) + ) num_tokens_for_logprob = sum( # We should have at least 1 token for sample in every case. max(extend_len - logprob_start_len, 1) @@ -641,6 +672,7 @@ def prepare_mlp_sync_batch_raw( # Gathered unconditionally like the other columns, so a per-process # metrics flag cannot make ranks disagree on what the tensor holds. attention_pairs=_local_attention_pairs(local_batch), + decoder_trim_rows=decoder_trim_rows, ) if num_dp_ranks == 1: diff --git a/python/sglang/srt/model_executor/encoder_swa_replay.py b/python/sglang/srt/model_executor/encoder_swa_replay.py index 81ed4e15f0..5f14cd2f5e 100644 --- a/python/sglang/srt/model_executor/encoder_swa_replay.py +++ b/python/sglang/srt/model_executor/encoder_swa_replay.py @@ -52,6 +52,23 @@ def folded_replay_rows(batch) -> int: ) +def decoder_swa_trim_rows(batch, *, folds_encoder: bool) -> int: + """Rows removed by a plain extend's decoder tail; -1 requests the backend + exchange when prompt outputs require whole-extend rows.""" + if not batch.forward_mode.is_extend_without_speculative(): + return 0 + if batch.return_logprob or batch.return_hidden_states: + return -1 + lens = list(batch.extend_lens) + if folds_encoder and batch.encoder_swa_reset is not None: + for i, (reset, prefix) in enumerate( + zip(batch.encoder_swa_reset, batch.prefix_lens) + ): + if reset: + lens[i] += replay_len(prefix) + return sum(max(0, n - 128) for n in lens) + + def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None: forward_batch.encoder_swa_row_floor = folded.row_floor forward_batch.encoder_swa_compress_rows = folded.keep_rows diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index d6f838f395..97cabd76fa 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -51,6 +51,7 @@ from sglang.srt.layers.dp_attention import ( set_is_extend_in_batch, world_dp_gather_enabled, ) +from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.model_executor.forward_batch_deepseek_mha_mixin import ( ForwardBatchDeepSeekMHAMixin, ) @@ -59,6 +60,7 @@ from sglang.srt.runtime_context import ( get_flags, get_lora, get_parallel, + get_platform, mamba_cache_chunk_size, ) from sglang.srt.speculative.spec_info import SpecInputType @@ -694,6 +696,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # Has to be None when cuda graph is captured. global_num_tokens_for_logprob_cpu: Optional[List[int]] = None global_num_tokens_for_logprob_gpu: Optional[torch.Tensor] = None + # Per DP rank, the rows the decoder SWA tail drops before the late layers; + # None when no rank trims (see DeepseekV4Model's late-layer DP switch). + global_decoder_trim_rows_cpu: Optional[List[int]] = None # Real (non-padding) token count, held at two scopes whose meaning never # changes once set: @@ -1189,6 +1194,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): ret.init_mlp_sync_metadata( batch, device, is_draft_worker=model_runner.is_draft_worker ) + if not model_runner.is_draft_worker: + ret.global_decoder_trim_rows_cpu = batch.global_decoder_trim_rows if ret.forward_mode.is_idle(): ret.positions = torch.empty((0,), dtype=torch.int64, device=device) @@ -1968,13 +1975,29 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): ): local_late_rows = model_runner.attn_backend.late_layer_num_tokens(self) if local_late_rows is not None: - late_rows = torch.empty(sync_group_size, dtype=torch.int64) - all_gather_single( - late_rows, - torch.tensor([local_late_rows], dtype=torch.int64), - group=get_parallel().tp_group.cpu_group, - ) - self.global_late_layer_num_tokens_cpu = late_rows.tolist() + trims = self.global_decoder_trim_rows_cpu + if ( + trims is not None + and get_platform().is_cuda + and get_parallel().attn_tp_size == 1 + and get_exec().graph.cuda_graph_config.prefill.backend + == Backend.DISABLED + and not self.dp_padding_mode.is_max_len() + ): + counts = [ + n - t for n, t in zip(global_num_tokens, trims, strict=True) + ] + assert min(counts) >= 0, (global_num_tokens, trims) + assert counts[dp_slot_in(counts)] == local_late_rows + self.global_late_layer_num_tokens_cpu = counts + else: + late_rows = torch.empty(sync_group_size, dtype=torch.int64) + all_gather_single( + late_rows, + torch.tensor([local_late_rows], dtype=torch.int64), + group=get_parallel().tp_group.cpu_group, + ) + self.global_late_layer_num_tokens_cpu = late_rows.tolist() TboForwardBatchPreparer.prepare( batch=self, is_draft_worker=model_runner.is_draft_worker diff --git a/test/registered/unit/model_executor/test_decoder_swa_scheduler_counts.py b/test/registered/unit/model_executor/test_decoder_swa_scheduler_counts.py new file mode 100644 index 0000000000..08b95cbe16 --- /dev/null +++ b/test/registered/unit/model_executor/test_decoder_swa_scheduler_counts.py @@ -0,0 +1,181 @@ +"""Decoder tail sizes reuse the scheduler exchange on eager CUDA DP steps.""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +import torch +from sglang.srt.managers.scheduler_components import dp_attn +from sglang.srt.managers.scheduler_components.dp_attn import _update_gather_batch +from sglang.srt.model_executor import forward_batch_info as fbi +from sglang.srt.model_executor.cuda_graph_config import Backend +from sglang.srt.model_executor.encoder_swa_replay import decoder_swa_trim_rows +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +@pytest.mark.parametrize("folds,expected", [(False, 3968), (True, 3998)]) +def test_trim_includes_folded_encoder_rows(folds, expected): + batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + extend_lens=[4096, 30, 40], + prefix_lens=[0, 512, 64], + encoder_swa_reset=[True, True, True], + return_logprob=False, + return_hidden_states=False, + ) + assert decoder_swa_trim_rows(batch, folds_encoder=folds) == expected + batch.return_logprob = True + assert decoder_swa_trim_rows(batch, folds_encoder=folds) == -1 + batch.return_logprob = False + batch.return_hidden_states = True + assert decoder_swa_trim_rows(batch, folds_encoder=folds) == -1 + + +@pytest.mark.parametrize( + "mode", [ForwardMode.DECODE, ForwardMode.IDLE, ForwardMode.TARGET_VERIFY] +) +def test_non_extend_peers_do_not_trim(mode): + assert ( + decoder_swa_trim_rows(SimpleNamespace(forward_mode=mode), folds_encoder=True) + == 0 + ) + + +@pytest.mark.parametrize( + "trims,expected", + [ + ([3968, 30, 0, 0], [3968, 30, 0, 0]), + ([0, 0, 0, 0], None), + ([3968, -1, 0, 0], None), + ], +) +def test_whole_output_peer_falls_back_on_every_rank(trims, expected): + info = MagicMock() + info.global_decoder_trim_rows = trims + batch = SimpleNamespace() + _update_gather_batch(batch, info, True, skip_global_metadata=True) + assert batch.global_decoder_trim_rows == expected + + +@pytest.mark.parametrize( + "rank,mode", + [ + (0, ForwardMode.EXTEND), + (1, ForwardMode.EXTEND), + (2, ForwardMode.IDLE), + (3, ForwardMode.DECODE), + ], +) +@pytest.mark.parametrize("fallback", [None, "graph", "hip", "whole_outputs"]) +def test_preparation_uses_scheduler_counts_or_preserves_backend_exchange( + rank, mode, fallback +): + rows, trims, late = [4096, 282, 0, 130], [3968, 30, 0, 0], [128, 252, 0, 130] + n = rows[rank] + batch = ForwardBatch( + forward_mode=mode, + batch_size=1 if n else 0, + input_ids=torch.zeros(n, dtype=torch.long), + req_pool_indices=torch.zeros(1 if n else 0, dtype=torch.long), + seq_lens=torch.tensor([n] if n else []), + out_cache_loc=torch.arange(n), + seq_lens_sum=n, + positions=torch.arange(n), + is_extend_in_batch=True, + global_num_tokens_cpu=rows, + global_num_tokens_for_logprob_cpu=rows, + global_num_tokens_gpu=torch.zeros(4, dtype=torch.int64), + global_decoder_trim_rows_cpu=None if fallback == "whole_outputs" else trims, + ) + runner = MagicMock(is_draft_worker=False) + runner.attn_tp_sequence_sharded.return_value = False + runner.attn_backend.late_layer_num_tokens.return_value = late[rank] + cfg = SimpleNamespace( + graph=SimpleNamespace( + cuda_graph_config=SimpleNamespace( + prefill=SimpleNamespace( + bs=[], + backend=Backend.BREAKABLE + if fallback == "graph" + else Backend.DISABLED, + ) + ) + ) + ) + + def exchange(output, local, **kwargs): + assert local.item() == late[rank] + output.copy_(torch.tensor(late)) + + with ( + patch.multiple( + fbi, + get_parallel=lambda: SimpleNamespace( + attn_tp_size=1, tp_group=SimpleNamespace(cpu_group=None) + ), + get_exec=lambda: cfg, + get_platform=lambda: SimpleNamespace(is_cuda=fallback != "hip"), + _elastic_should_preserve_local_token_counts=lambda **kwargs: False, + _mega_moe_materializes_idle_rank=lambda batch: False, + dp_slot_in=lambda counts: rank, + set_dp_buffer_len_from_batch=lambda *args: None, + set_is_extend_in_batch=lambda *args: None, + _is_cpu=True, + ), + patch.object(fbi, "mambaish_config", return_value=None, create=True), + patch.object( + fbi.DpPaddingMode, + "get_dp_padding_mode", + return_value=fbi.DpPaddingMode.SUM_LEN, + ), + patch.object(ForwardBatch, "_pad_inputs_to_size"), + patch( + "sglang.srt.batch_overlap.two_batch_overlap.TboForwardBatchPreparer.prepare" + ), + patch.object(fbi, "all_gather_single", side_effect=exchange) as gather, + ): + batch.prepare_mlp_sync_batch(runner) + assert batch.global_late_layer_num_tokens_cpu == late + assert gather.call_count == (0 if fallback is None else 1) + + +def test_scheduler_exchange_carries_peer_trims_and_inactive_slots(): + infos = [ + dp_attn.MLPSyncBatchInfo( + num_dp_ranks=4, + tp_size=1, + cp_size=1, + num_tokens=n, + num_tokens_for_logprob=n, + can_run_decode_cuda_graph=False, + can_run_draft_cuda_graph=False, + can_run_prefill_cuda_graph=False, + is_extend_in_batch=True, + local_can_run_tbo=False, + local_forward_mode=ForwardMode.EXTEND.value, + decoder_trim_rows=t, + ) + for n, t in zip([4096, 282, 0, 130], [3968, 30, 0, 0]) + ] + gathered = torch.cat([info._get_local_tensor("cpu") for info in infos]) + with ( + patch.object( + dp_attn, + "exchange_sync_rows", + side_effect=lambda out, *a, **kw: out.copy_(gathered), + ), + patch.object( + dp_attn, + "get_parallel", + return_value=SimpleNamespace( + tp_group=SimpleNamespace(active_ranks_cpu=torch.tensor([1, 1, 0, 1])) + ), + ), + ): + for info in infos: + info.all_gather(device="cpu", group=None) + assert info.global_decoder_trim_rows == [3968, 30, 0, 0] + assert info.global_num_tokens == [4096, 282, 0, 130] diff --git a/test/registered/unit/spec/test_dp_spec_prefill_coordination.py b/test/registered/unit/spec/test_dp_spec_prefill_coordination.py index 6eab95252f..f4c810aa6a 100644 --- a/test/registered/unit/spec/test_dp_spec_prefill_coordination.py +++ b/test/registered/unit/spec/test_dp_spec_prefill_coordination.py @@ -133,6 +133,7 @@ class TestDPSpecPrefillCoordinationPlan(CustomTestCase): num_tokens_for_logprob=32, global_num_tokens=[32, 7], global_num_tokens_for_logprob=[32, 7], + global_decoder_trim_rows=[0, 0], tp0_info_cpu=torch.tensor([[32, 32, 1, 0], [7, 7, 1, 0]]), is_extend_in_batch=False, tbo_split_seq_index=None,