diff --git a/python/sglang/srt/layers/attention/base_attn_backend.py b/python/sglang/srt/layers/attention/base_attn_backend.py index 76c88b6ba5..03e16a0bc6 100644 --- a/python/sglang/srt/layers/attention/base_attn_backend.py +++ b/python/sglang/srt/layers/attention/base_attn_backend.py @@ -245,6 +245,11 @@ class AttentionBackend(ABC): """Get the fill value for padded seq lens. Typically, it is 0 or 1.""" raise NotImplementedError() + def late_layer_num_tokens(self, forward_batch: ForwardBatch) -> Optional[int]: + """Rows this rank runs from a late-layer row switch on (DeepSeek-V4.1 + decoder SWA bounded replay); None for backends without one.""" + return None + def on_after_cuda_graph_warmup(self): """Hook between cuda graph warmup pass and the actual capture. diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 9dbdfeb31b..9ef4e245be 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -989,6 +989,19 @@ def late_layer_whole_extend( ) +def late_layer_num_tokens(forward_batch: ForwardBatch, *, dspark: bool) -> int: + """Rows a rank runs from late_layer_start on: an extend's tail rows, as + late_layer_tail_layout picks them, and every row otherwise.""" + if not forward_batch.forward_mode.is_extend_without_speculative(): + return forward_batch.input_ids.shape[0] + whole_extend = late_layer_whole_extend(forward_batch, dspark=dspark) + return sum( + late_layer_tail_lens( + forward_batch.extend_seq_lens_cpu, SWA_WINDOW, whole_extend + ) + ) + + def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor: assert t.shape[0] <= num_rows, (t.shape[0], num_rows) if t.shape[0] == num_rows: @@ -1640,6 +1653,13 @@ class DeepseekV4AttnBackend( return False return True + def late_layer_num_tokens(self, forward_batch: ForwardBatch) -> Optional[int]: + if not self.enable_decoder_swa_bounded_replay: + return None + return late_layer_num_tokens( + forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark() + ) + def vote_prefill_cuda_graph(self, batch: ScheduleBatch) -> bool: """`can_run_prefill_cuda_graph`'s late-layer tail check judged from the schedule batch, for the DP prefill-graph vote: no rank replays the graph diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index 21e7ee6adb..ee2083cb97 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -68,6 +68,7 @@ from sglang.srt.layers.attention.deepseek_v4_backend import ( _as_int_list, _prefill_graph_max_seq_len, _tail_rows, + late_layer_num_tokens, late_layer_whole_extend, ) from sglang.srt.layers.attention.dsv4.compressor_v2 import ( @@ -2176,6 +2177,11 @@ class DeepseekV4HipRadixBackend( else None ) + def late_layer_num_tokens(self, forward_batch: ForwardBatch) -> Optional[int]: + if not self.enable_decoder_swa_bounded_replay or self.is_draft_worker: + return None + return late_layer_num_tokens(forward_batch, dspark=self.is_dspark) + def _init_forward_metadata_prefill_from_batch( self, forward_batch: ForwardBatch, diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 8cdb79d1ec..dae1ff02ab 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -39,6 +39,7 @@ import torch from sglang.kernels.ops.attention.clamp_position import clamp_position from sglang.kernels.ops.attention.position import compute_position_triton from sglang.srt.configs.hybrid_arch import mambaish_config +from sglang.srt.distributed.utils import all_gather_single from sglang.srt.environ import envs from sglang.srt.kv_canary.req_to_expected_token_ids_manager import ( compute_req_all_ids_info, @@ -749,6 +750,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # global_num_tokens_cpu as published for the DP gather: attn-TP aligned and, # under MAX_LEN, padded to the max. None when the raw list already is. global_num_tokens_padded_cpu: Optional[List[int]] = None + # Each DP rank's rows from a late-layer row switch on; None on steps that skip + # the exchange. + global_late_layer_num_tokens_cpu: Optional[List[int]] = None # For Qwen2-VL mrope_positions: torch.Tensor = None @@ -1857,6 +1861,22 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): self.global_num_tokens_gpu.copy_( global_num_tokens_pinned, non_blocking=self.use_pin_memory ) + # Exchanged here, not in the model forward: every rank reaches this point + # after reshaping its batch and before it picks a prefill graph or eager. + if ( + self.is_extend_in_batch + and sync_group_size > 1 + and not model_runner.is_draft_worker + ): + local_late_rows = model_runner.attn_backend.late_layer_num_tokens(self) + if local_late_rows is not None: + late_rows = torch.empty(sync_group_size, dtype=torch.int64) + all_gather_single( + late_rows, + torch.tensor([local_late_rows], dtype=torch.int64), + group=get_parallel().tp_group.cpu_group, + ) + self.global_late_layer_num_tokens_cpu = late_rows.tolist() TboForwardBatchPreparer.prepare( batch=self, is_draft_worker=model_runner.is_draft_worker diff --git a/python/sglang/srt/models/deepseek_common/decoder_swa_bounded_replay.py b/python/sglang/srt/models/deepseek_common/decoder_swa_bounded_replay.py index 07e6f700a5..2feb9fa117 100644 --- a/python/sglang/srt/models/deepseek_common/decoder_swa_bounded_replay.py +++ b/python/sglang/srt/models/deepseek_common/decoder_swa_bounded_replay.py @@ -26,11 +26,20 @@ def late_layer_dp_counts( # Local-only counts (MegaMoE with a DP LM head): no peer geometry to share. counts = [local_late_rows] elif in_graph: - # Prefill graphs pad every DP rank to one shape, so the tail block is - # rank-uniform and the capture bakes it in; replays never run this code. + # The capture bakes the tail block and replays never run this code. Every + # rank shares one bucket whenever the graph holds a DP gather. counts = [local_late_rows] * len(early) + elif forward_batch.global_late_layer_num_tokens_cpu is not None: + # Exchanged in ForwardBatch.prepare_mlp_sync_batch, which a peer replaying + # the prefill graph reaches too; a gather here would wait for it forever. + counts = forward_batch.global_late_layer_num_tokens_cpu + assert counts[get_parallel().attn_dp_rank] == local_late_rows, ( + counts, + local_late_rows, + ) else: - # Tail row counts are host-known; the CPU group avoids a device sync. + # Without the exchange (no extend in the batch) every rank takes this + # path; the CPU group avoids a device sync. counts = torch.empty(len(early), dtype=torch.int64) all_gather_single( counts, diff --git a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py index 82c96a4fc3..2d81d6bdcd 100644 --- a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py +++ b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py @@ -1,9 +1,9 @@ -"""Decoder SWA bounded replay: requests that read prompt rows (prompt logprobs, -every hidden state) run their whole extend through the late layers. CPU only.""" +"""Decoder SWA bounded replay: the rows the late layers run (whole extends for +requests that read prompt rows) and how DP ranks learn each other's. CPU only.""" import unittest from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import MagicMock, patch import torch @@ -16,8 +16,19 @@ from sglang.srt.layers.attention.deepseek_v4_backend import ( DeepseekV4AttnBackend, late_layer_whole_extend, ) -from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode +from sglang.srt.model_executor import forward_batch_info +from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseConfig +from sglang.srt.model_executor.forward_batch_info import ( + CaptureHiddenMode, + ForwardBatch, + ForwardMode, +) +from sglang.srt.models.deepseek_common import decoder_swa_bounded_replay +from sglang.srt.models.deepseek_common.decoder_swa_bounded_replay import ( + late_layer_dp_counts, +) from sglang.srt.models.deepseek_v4 import DeepseekV4Model +from sglang.srt.runtime_context import get_context, get_parallel from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -173,5 +184,92 @@ class TestTailReadersCheck(CustomTestCase): self._check(_batch([300], capture=CaptureHiddenMode.FULL), [128]) +class TestLateLayerRowsUnderDpAttention(CustomTestCase): + """A rank replaying the prefill graph never runs the model's late-layer code, + so its eager peers learn its late-layer rows before the forward.""" + + def setUp(self): + override = get_context().override_server_args( + tp_size=2, + dp_size=2, + enable_dp_attention=True, + cuda_graph_config=CudaGraphConfig(prefill=PhaseConfig(bs=[])), + ) + override.install() + self.addCleanup(override.restore) + + def _dp_rank(self, rank): + return get_parallel().override( + attn_tp_size=1, + attn_dp_size=2, + attn_dp_rank=rank, + tp_group=SimpleNamespace(cpu_group=None), + ) + + def test_rows_are_exchanged_before_the_forward(self): + # Rank 0 prefills 300 rows; rank 1 is idle. + n = 300 + fb = ForwardBatch( + forward_mode=ForwardMode.EXTEND, + batch_size=1, + input_ids=torch.arange(n), + req_pool_indices=torch.tensor([1]), + seq_lens=torch.tensor([n]), + out_cache_loc=torch.arange(n), + seq_lens_sum=n, + positions=torch.arange(n), + seq_lens_cpu=torch.tensor([n]), + extend_num_tokens=n, + extend_seq_lens=torch.tensor([n]), + extend_seq_lens_cpu=[n], + extend_prefix_lens_cpu=[0], + is_extend_in_batch=True, + ) + fb.init_mlp_sync_metadata( + SimpleNamespace( + global_num_tokens=[n, 0], + global_num_tokens_for_logprob=[1, 0], + dp_spec_prefill_coordination_applied=False, + draft_global_num_tokens=None, + can_run_decode_cuda_graph=False, + can_run_dp_draft_cuda_graph=False, + ), + CPU, + ) + backend = object.__new__(DeepseekV4AttnBackend) + backend.enable_decoder_swa_bounded_replay = True + backend.model_runner = SimpleNamespace(spec_algorithm=SpeculativeAlgorithm.NONE) + backend.get_cpu_graph_seq_len_fill_value = lambda: 1 + runner = MagicMock(attn_backend=backend, is_draft_worker=False) + runner.attn_tp_sequence_sharded.return_value = False + + def idle_peer(out, local, group): + out.copy_(torch.cat([local, local.new_zeros(1)])) + + with ( + self._dp_rank(0), + # CPU runners have no driver to pin the synced token counts with. + patch.object(forward_batch_info, "_is_cpu", True), + patch.object( + forward_batch_info, "all_gather_single", side_effect=idle_peer + ), + ): + fb.prepare_mlp_sync_batch(runner) + self.assertEqual(fb.global_late_layer_num_tokens_cpu, [SWA_WINDOW, 0]) + + for rank, local_rows in ((0, SWA_WINDOW), (1, 0)): + with ( + self.subTest(rank=rank), + self._dp_rank(rank), + patch.object( + decoder_swa_bounded_replay, + "all_gather_single", + side_effect=AssertionError("a collective inside the forward"), + ), + ): + counts = late_layer_dp_counts(fb, local_rows, in_graph=False) + self.assertEqual(counts, [SWA_WINDOW, 0]) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/model_executor/test_mlp_sync_hybrid_target_verify.py b/test/registered/unit/model_executor/test_mlp_sync_hybrid_target_verify.py index 2804a0864f..7e4298042c 100644 --- a/test/registered/unit/model_executor/test_mlp_sync_hybrid_target_verify.py +++ b/test/registered/unit/model_executor/test_mlp_sync_hybrid_target_verify.py @@ -41,6 +41,7 @@ def _sync(forward_mode, num_reqs, global_num_tokens): ) model_runner = MagicMock(is_draft_worker=False) model_runner.attn_tp_sequence_sharded.return_value = False + model_runner.attn_backend.late_layer_num_tokens.return_value = None no_prefill_graph = SimpleNamespace( graph=SimpleNamespace( cuda_graph_config=SimpleNamespace(prefill=SimpleNamespace(bs=[]))