From 97b79f0796687b26335d3b2d4c051f6c830f53db Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 22 Oct 2026 17:36:00 -0700 Subject: [PATCH] [Fix] prefill-graph: Keep every DP rank eager when the replay tail captures behind rank barriers (#45881) --- python/sglang/srt/layers/dp_attention.py | 7 +++++++ .../model_executor/runner/prefill_cuda_graph_runner.py | 9 ++++++--- python/sglang/srt/models/deepseek_v4_replay_tail.py | 7 ++++++- python/sglang/srt/runtime_context.py | 3 +++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py index 022f29efd1..86e69bc0d4 100644 --- a/python/sglang/srt/layers/dp_attention.py +++ b/python/sglang/srt/layers/dp_attention.py @@ -941,6 +941,13 @@ def _note_dp_gather_in_prefill_graph() -> None: dp.prefill_graph_has_dp_gather = True +def note_rank_barrier_in_prefill_graph() -> None: + """For breaks that capture graphs of their own behind TP-group barriers.""" + dp = get_flags().dp + if dp.capturing_prefill_graph: + dp.prefill_graph_has_rank_barrier = True + + def _dp_gather( global_tokens: torch.Tensor, local_tokens: torch.Tensor, diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index affdd79198..fb17965a83 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -979,12 +979,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): # DSV4 DP attention / DeepEP collectives need every DP rank to enter # the same replay path. Sparse-DP batches (one or more ranks with # zero local tokens) fall back to eager to avoid hanging ranks. - # MegaMoE graphs without a captured DP gather tolerate per-rank buckets - # and an eager idle rank; graphs with one fall through to the check. + # MegaMoE graphs without a captured DP gather or rank barrier tolerate + # per-rank buckets and an eager idle rank; others fall through. global_num_tokens = forward_batch.global_num_tokens_cpu if global_num_tokens is None: return False - if prefill_graph_tolerates_sum_len(): + if ( + prefill_graph_tolerates_sum_len() + and not get_flags().dp.prefill_graph_has_rank_barrier + ): return False return len(global_num_tokens) > 1 and any( int(num_tokens) == 0 for num_tokens in global_num_tokens diff --git a/python/sglang/srt/models/deepseek_v4_replay_tail.py b/python/sglang/srt/models/deepseek_v4_replay_tail.py index b5e5529298..440f1186e3 100644 --- a/python/sglang/srt/models/deepseek_v4_replay_tail.py +++ b/python/sglang/srt/models/deepseek_v4_replay_tail.py @@ -20,7 +20,10 @@ import torch from sglang.srt.distributed.utils import all_gather_single from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder -from sglang.srt.layers.dp_attention import dp_gather_replicate +from sglang.srt.layers.dp_attention import ( + dp_gather_replicate, + note_rank_barrier_in_prefill_graph, +) from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.moe import get_moe_a2a_backend from sglang.srt.model_executor.cuda_graph_config import ( @@ -650,6 +653,8 @@ def _tail_layers_in_prefill_graph( input_ids: torch.Tensor, input_ids_global: torch.Tensor, ) -> Tuple[Optional[torch.Tensor], ...]: + # The tail captures its row-bucket graphs on first use. + note_rank_barrier_in_prefill_graph() backend = get_attn_backend() saved, forward_batch = _enter_eager_tail_in_prefill_graph( backend, get_tc_piecewise_forward_context().forward_batch, source_kv_layer_id diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 90c16db50b..054e46f338 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -597,6 +597,9 @@ class DpFlags(_FlagGroupBase): # gather/scatter helpers, whose captured geometry needs one shared bucket. capturing_prefill_graph: bool = False prefill_graph_has_dp_gather: bool = False + # Set during prefill-graph capture by breaks that capture graphs of their own + # behind TP-group barriers; every DP rank must then take the same path. + prefill_graph_has_rank_barrier: bool = False # This process's slot in the target's DP sync while a draft scope narrows # attention; dp_gather_width() also reads it as the in-scope marker. scoped_gather_slot: Optional[int] = None