[Fix] deepseek-v4.1: Exchange late-layer DP row counts before the forward (#44948)
This commit is contained in:
@@ -245,6 +245,11 @@ class AttentionBackend(ABC):
|
||||
"""Get the fill value for padded seq lens. Typically, it is 0 or 1."""
|
||||
raise NotImplementedError()
|
||||
|
||||
def late_layer_num_tokens(self, forward_batch: ForwardBatch) -> Optional[int]:
|
||||
"""Rows this rank runs from a late-layer row switch on (DeepSeek-V4.1
|
||||
decoder SWA bounded replay); None for backends without one."""
|
||||
return None
|
||||
|
||||
def on_after_cuda_graph_warmup(self):
|
||||
"""Hook between cuda graph warmup pass and the actual capture.
|
||||
|
||||
|
||||
@@ -989,6 +989,19 @@ def late_layer_whole_extend(
|
||||
)
|
||||
|
||||
|
||||
def late_layer_num_tokens(forward_batch: ForwardBatch, *, dspark: bool) -> int:
|
||||
"""Rows a rank runs from late_layer_start on: an extend's tail rows, as
|
||||
late_layer_tail_layout picks them, and every row otherwise."""
|
||||
if not forward_batch.forward_mode.is_extend_without_speculative():
|
||||
return forward_batch.input_ids.shape[0]
|
||||
whole_extend = late_layer_whole_extend(forward_batch, dspark=dspark)
|
||||
return sum(
|
||||
late_layer_tail_lens(
|
||||
forward_batch.extend_seq_lens_cpu, SWA_WINDOW, whole_extend
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor:
|
||||
assert t.shape[0] <= num_rows, (t.shape[0], num_rows)
|
||||
if t.shape[0] == num_rows:
|
||||
@@ -1640,6 +1653,13 @@ class DeepseekV4AttnBackend(
|
||||
return False
|
||||
return True
|
||||
|
||||
def late_layer_num_tokens(self, forward_batch: ForwardBatch) -> Optional[int]:
|
||||
if not self.enable_decoder_swa_bounded_replay:
|
||||
return None
|
||||
return late_layer_num_tokens(
|
||||
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
|
||||
)
|
||||
|
||||
def vote_prefill_cuda_graph(self, batch: ScheduleBatch) -> bool:
|
||||
"""`can_run_prefill_cuda_graph`'s late-layer tail check judged from the
|
||||
schedule batch, for the DP prefill-graph vote: no rank replays the graph
|
||||
|
||||
@@ -68,6 +68,7 @@ from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
_as_int_list,
|
||||
_prefill_graph_max_seq_len,
|
||||
_tail_rows,
|
||||
late_layer_num_tokens,
|
||||
late_layer_whole_extend,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.compressor_v2 import (
|
||||
@@ -2176,6 +2177,11 @@ class DeepseekV4HipRadixBackend(
|
||||
else None
|
||||
)
|
||||
|
||||
def late_layer_num_tokens(self, forward_batch: ForwardBatch) -> Optional[int]:
|
||||
if not self.enable_decoder_swa_bounded_replay or self.is_draft_worker:
|
||||
return None
|
||||
return late_layer_num_tokens(forward_batch, dspark=self.is_dspark)
|
||||
|
||||
def _init_forward_metadata_prefill_from_batch(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
|
||||
@@ -39,6 +39,7 @@ import torch
|
||||
from sglang.kernels.ops.attention.clamp_position import clamp_position
|
||||
from sglang.kernels.ops.attention.position import compute_position_triton
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.distributed.utils import all_gather_single
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.kv_canary.req_to_expected_token_ids_manager import (
|
||||
compute_req_all_ids_info,
|
||||
@@ -749,6 +750,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# global_num_tokens_cpu as published for the DP gather: attn-TP aligned and,
|
||||
# under MAX_LEN, padded to the max. None when the raw list already is.
|
||||
global_num_tokens_padded_cpu: Optional[List[int]] = None
|
||||
# Each DP rank's rows from a late-layer row switch on; None on steps that skip
|
||||
# the exchange.
|
||||
global_late_layer_num_tokens_cpu: Optional[List[int]] = None
|
||||
|
||||
# For Qwen2-VL
|
||||
mrope_positions: torch.Tensor = None
|
||||
@@ -1857,6 +1861,22 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
self.global_num_tokens_gpu.copy_(
|
||||
global_num_tokens_pinned, non_blocking=self.use_pin_memory
|
||||
)
|
||||
# Exchanged here, not in the model forward: every rank reaches this point
|
||||
# after reshaping its batch and before it picks a prefill graph or eager.
|
||||
if (
|
||||
self.is_extend_in_batch
|
||||
and sync_group_size > 1
|
||||
and not model_runner.is_draft_worker
|
||||
):
|
||||
local_late_rows = model_runner.attn_backend.late_layer_num_tokens(self)
|
||||
if local_late_rows is not None:
|
||||
late_rows = torch.empty(sync_group_size, dtype=torch.int64)
|
||||
all_gather_single(
|
||||
late_rows,
|
||||
torch.tensor([local_late_rows], dtype=torch.int64),
|
||||
group=get_parallel().tp_group.cpu_group,
|
||||
)
|
||||
self.global_late_layer_num_tokens_cpu = late_rows.tolist()
|
||||
|
||||
TboForwardBatchPreparer.prepare(
|
||||
batch=self, is_draft_worker=model_runner.is_draft_worker
|
||||
|
||||
@@ -26,11 +26,20 @@ def late_layer_dp_counts(
|
||||
# Local-only counts (MegaMoE with a DP LM head): no peer geometry to share.
|
||||
counts = [local_late_rows]
|
||||
elif in_graph:
|
||||
# Prefill graphs pad every DP rank to one shape, so the tail block is
|
||||
# rank-uniform and the capture bakes it in; replays never run this code.
|
||||
# The capture bakes the tail block and replays never run this code. Every
|
||||
# rank shares one bucket whenever the graph holds a DP gather.
|
||||
counts = [local_late_rows] * len(early)
|
||||
elif forward_batch.global_late_layer_num_tokens_cpu is not None:
|
||||
# Exchanged in ForwardBatch.prepare_mlp_sync_batch, which a peer replaying
|
||||
# the prefill graph reaches too; a gather here would wait for it forever.
|
||||
counts = forward_batch.global_late_layer_num_tokens_cpu
|
||||
assert counts[get_parallel().attn_dp_rank] == local_late_rows, (
|
||||
counts,
|
||||
local_late_rows,
|
||||
)
|
||||
else:
|
||||
# Tail row counts are host-known; the CPU group avoids a device sync.
|
||||
# Without the exchange (no extend in the batch) every rank takes this
|
||||
# path; the CPU group avoids a device sync.
|
||||
counts = torch.empty(len(early), dtype=torch.int64)
|
||||
all_gather_single(
|
||||
counts,
|
||||
|
||||
Reference in New Issue
Block a user