[Fix] deepseek-v4.1: Exchange late-layer DP row counts before the forward (#44948)

This commit is contained in:
Liangsheng Yin
2026-10-17 21:46:46 -07:00
committed by GitHub
parent 6fe630ae14
commit d20d010574
7 changed files with 166 additions and 7 deletions
@@ -245,6 +245,11 @@ class AttentionBackend(ABC):
"""Get the fill value for padded seq lens. Typically, it is 0 or 1."""
raise NotImplementedError()
def late_layer_num_tokens(self, forward_batch: ForwardBatch) -> Optional[int]:
"""Rows this rank runs from a late-layer row switch on (DeepSeek-V4.1
decoder SWA bounded replay); None for backends without one."""
return None
def on_after_cuda_graph_warmup(self):
"""Hook between cuda graph warmup pass and the actual capture.
@@ -989,6 +989,19 @@ def late_layer_whole_extend(
)
def late_layer_num_tokens(forward_batch: ForwardBatch, *, dspark: bool) -> int:
"""Rows a rank runs from late_layer_start on: an extend's tail rows, as
late_layer_tail_layout picks them, and every row otherwise."""
if not forward_batch.forward_mode.is_extend_without_speculative():
return forward_batch.input_ids.shape[0]
whole_extend = late_layer_whole_extend(forward_batch, dspark=dspark)
return sum(
late_layer_tail_lens(
forward_batch.extend_seq_lens_cpu, SWA_WINDOW, whole_extend
)
)
def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor:
assert t.shape[0] <= num_rows, (t.shape[0], num_rows)
if t.shape[0] == num_rows:
@@ -1640,6 +1653,13 @@ class DeepseekV4AttnBackend(
return False
return True
def late_layer_num_tokens(self, forward_batch: ForwardBatch) -> Optional[int]:
if not self.enable_decoder_swa_bounded_replay:
return None
return late_layer_num_tokens(
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
)
def vote_prefill_cuda_graph(self, batch: ScheduleBatch) -> bool:
"""`can_run_prefill_cuda_graph`'s late-layer tail check judged from the
schedule batch, for the DP prefill-graph vote: no rank replays the graph
@@ -68,6 +68,7 @@ from sglang.srt.layers.attention.deepseek_v4_backend import (
_as_int_list,
_prefill_graph_max_seq_len,
_tail_rows,
late_layer_num_tokens,
late_layer_whole_extend,
)
from sglang.srt.layers.attention.dsv4.compressor_v2 import (
@@ -2176,6 +2177,11 @@ class DeepseekV4HipRadixBackend(
else None
)
def late_layer_num_tokens(self, forward_batch: ForwardBatch) -> Optional[int]:
if not self.enable_decoder_swa_bounded_replay or self.is_draft_worker:
return None
return late_layer_num_tokens(forward_batch, dspark=self.is_dspark)
def _init_forward_metadata_prefill_from_batch(
self,
forward_batch: ForwardBatch,
@@ -39,6 +39,7 @@ import torch
from sglang.kernels.ops.attention.clamp_position import clamp_position
from sglang.kernels.ops.attention.position import compute_position_triton
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.distributed.utils import all_gather_single
from sglang.srt.environ import envs
from sglang.srt.kv_canary.req_to_expected_token_ids_manager import (
compute_req_all_ids_info,
@@ -749,6 +750,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# global_num_tokens_cpu as published for the DP gather: attn-TP aligned and,
# under MAX_LEN, padded to the max. None when the raw list already is.
global_num_tokens_padded_cpu: Optional[List[int]] = None
# Each DP rank's rows from a late-layer row switch on; None on steps that skip
# the exchange.
global_late_layer_num_tokens_cpu: Optional[List[int]] = None
# For Qwen2-VL
mrope_positions: torch.Tensor = None
@@ -1857,6 +1861,22 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
self.global_num_tokens_gpu.copy_(
global_num_tokens_pinned, non_blocking=self.use_pin_memory
)
# Exchanged here, not in the model forward: every rank reaches this point
# after reshaping its batch and before it picks a prefill graph or eager.
if (
self.is_extend_in_batch
and sync_group_size > 1
and not model_runner.is_draft_worker
):
local_late_rows = model_runner.attn_backend.late_layer_num_tokens(self)
if local_late_rows is not None:
late_rows = torch.empty(sync_group_size, dtype=torch.int64)
all_gather_single(
late_rows,
torch.tensor([local_late_rows], dtype=torch.int64),
group=get_parallel().tp_group.cpu_group,
)
self.global_late_layer_num_tokens_cpu = late_rows.tolist()
TboForwardBatchPreparer.prepare(
batch=self, is_draft_worker=model_runner.is_draft_worker
@@ -26,11 +26,20 @@ def late_layer_dp_counts(
# Local-only counts (MegaMoE with a DP LM head): no peer geometry to share.
counts = [local_late_rows]
elif in_graph:
# Prefill graphs pad every DP rank to one shape, so the tail block is
# rank-uniform and the capture bakes it in; replays never run this code.
# The capture bakes the tail block and replays never run this code. Every
# rank shares one bucket whenever the graph holds a DP gather.
counts = [local_late_rows] * len(early)
elif forward_batch.global_late_layer_num_tokens_cpu is not None:
# Exchanged in ForwardBatch.prepare_mlp_sync_batch, which a peer replaying
# the prefill graph reaches too; a gather here would wait for it forever.
counts = forward_batch.global_late_layer_num_tokens_cpu
assert counts[get_parallel().attn_dp_rank] == local_late_rows, (
counts,
local_late_rows,
)
else:
# Tail row counts are host-known; the CPU group avoids a device sync.
# Without the exchange (no extend in the batch) every rank takes this
# path; the CPU group avoids a device sync.
counts = torch.empty(len(early), dtype=torch.int64)
all_gather_single(
counts,