[Fix] prefill-graph: Keep every DP rank eager when the replay tail captures behind rank barriers (#45881)
This commit is contained in:
@@ -941,6 +941,13 @@ def _note_dp_gather_in_prefill_graph() -> None:
|
||||
dp.prefill_graph_has_dp_gather = True
|
||||
|
||||
|
||||
def note_rank_barrier_in_prefill_graph() -> None:
|
||||
"""For breaks that capture graphs of their own behind TP-group barriers."""
|
||||
dp = get_flags().dp
|
||||
if dp.capturing_prefill_graph:
|
||||
dp.prefill_graph_has_rank_barrier = True
|
||||
|
||||
|
||||
def _dp_gather(
|
||||
global_tokens: torch.Tensor,
|
||||
local_tokens: torch.Tensor,
|
||||
|
||||
@@ -979,12 +979,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
# DSV4 DP attention / DeepEP collectives need every DP rank to enter
|
||||
# the same replay path. Sparse-DP batches (one or more ranks with
|
||||
# zero local tokens) fall back to eager to avoid hanging ranks.
|
||||
# MegaMoE graphs without a captured DP gather tolerate per-rank buckets
|
||||
# and an eager idle rank; graphs with one fall through to the check.
|
||||
# MegaMoE graphs without a captured DP gather or rank barrier tolerate
|
||||
# per-rank buckets and an eager idle rank; others fall through.
|
||||
global_num_tokens = forward_batch.global_num_tokens_cpu
|
||||
if global_num_tokens is None:
|
||||
return False
|
||||
if prefill_graph_tolerates_sum_len():
|
||||
if (
|
||||
prefill_graph_tolerates_sum_len()
|
||||
and not get_flags().dp.prefill_graph_has_rank_barrier
|
||||
):
|
||||
return False
|
||||
return len(global_num_tokens) > 1 and any(
|
||||
int(num_tokens) == 0 for num_tokens in global_num_tokens
|
||||
|
||||
@@ -20,7 +20,10 @@ import torch
|
||||
from sglang.srt.distributed.utils import all_gather_single
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.dp_attention import dp_gather_replicate
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
dp_gather_replicate,
|
||||
note_rank_barrier_in_prefill_graph,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
@@ -650,6 +653,8 @@ def _tail_layers_in_prefill_graph(
|
||||
input_ids: torch.Tensor,
|
||||
input_ids_global: torch.Tensor,
|
||||
) -> Tuple[Optional[torch.Tensor], ...]:
|
||||
# The tail captures its row-bucket graphs on first use.
|
||||
note_rank_barrier_in_prefill_graph()
|
||||
backend = get_attn_backend()
|
||||
saved, forward_batch = _enter_eager_tail_in_prefill_graph(
|
||||
backend, get_tc_piecewise_forward_context().forward_batch, source_kv_layer_id
|
||||
|
||||
@@ -597,6 +597,9 @@ class DpFlags(_FlagGroupBase):
|
||||
# gather/scatter helpers, whose captured geometry needs one shared bucket.
|
||||
capturing_prefill_graph: bool = False
|
||||
prefill_graph_has_dp_gather: bool = False
|
||||
# Set during prefill-graph capture by breaks that capture graphs of their own
|
||||
# behind TP-group barriers; every DP rank must then take the same path.
|
||||
prefill_graph_has_rank_barrier: bool = False
|
||||
# This process's slot in the target's DP sync while a draft scope narrows
|
||||
# attention; dp_gather_width() also reads it as the in-scope marker.
|
||||
scoped_gather_slot: Optional[int] = None
|
||||
|
||||
Reference in New Issue
Block a user