[Fix] prefill-graph: Keep every DP rank eager when the replay tail captures behind rank barriers (#45881)

This commit is contained in:
Liangsheng Yin
2026-10-22 17:36:00 -07:00
committed by GitHub
parent 9c7dd3c45f
commit 97b79f0796
4 changed files with 22 additions and 4 deletions
+7
View File
@@ -941,6 +941,13 @@ def _note_dp_gather_in_prefill_graph() -> None:
dp.prefill_graph_has_dp_gather = True
def note_rank_barrier_in_prefill_graph() -> None:
"""For breaks that capture graphs of their own behind TP-group barriers."""
dp = get_flags().dp
if dp.capturing_prefill_graph:
dp.prefill_graph_has_rank_barrier = True
def _dp_gather(
global_tokens: torch.Tensor,
local_tokens: torch.Tensor,
@@ -979,12 +979,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
# DSV4 DP attention / DeepEP collectives need every DP rank to enter
# the same replay path. Sparse-DP batches (one or more ranks with
# zero local tokens) fall back to eager to avoid hanging ranks.
# MegaMoE graphs without a captured DP gather tolerate per-rank buckets
# and an eager idle rank; graphs with one fall through to the check.
# MegaMoE graphs without a captured DP gather or rank barrier tolerate
# per-rank buckets and an eager idle rank; others fall through.
global_num_tokens = forward_batch.global_num_tokens_cpu
if global_num_tokens is None:
return False
if prefill_graph_tolerates_sum_len():
if (
prefill_graph_tolerates_sum_len()
and not get_flags().dp.prefill_graph_has_rank_barrier
):
return False
return len(global_num_tokens) > 1 and any(
int(num_tokens) == 0 for num_tokens in global_num_tokens
@@ -20,7 +20,10 @@ import torch
from sglang.srt.distributed.utils import all_gather_single
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dp_attention import dp_gather_replicate
from sglang.srt.layers.dp_attention import (
dp_gather_replicate,
note_rank_barrier_in_prefill_graph,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.model_executor.cuda_graph_config import (
@@ -650,6 +653,8 @@ def _tail_layers_in_prefill_graph(
input_ids: torch.Tensor,
input_ids_global: torch.Tensor,
) -> Tuple[Optional[torch.Tensor], ...]:
# The tail captures its row-bucket graphs on first use.
note_rank_barrier_in_prefill_graph()
backend = get_attn_backend()
saved, forward_batch = _enter_eager_tail_in_prefill_graph(
backend, get_tc_piecewise_forward_context().forward_batch, source_kv_layer_id
+3
View File
@@ -597,6 +597,9 @@ class DpFlags(_FlagGroupBase):
# gather/scatter helpers, whose captured geometry needs one shared bucket.
capturing_prefill_graph: bool = False
prefill_graph_has_dp_gather: bool = False
# Set during prefill-graph capture by breaks that capture graphs of their own
# behind TP-group barriers; every DP rank must then take the same path.
prefill_graph_has_rank_barrier: bool = False
# This process's slot in the target's DP sync while a draft scope narrows
# attention; dp_gather_width() also reads it as the in-scope marker.
scoped_gather_slot: Optional[int] = None