[Experimental] Preserve speculative decoding during prefill across DP ranks (#40118)

This commit is contained in:
Yuwei An
2026-09-24 00:01:12 -07:00
committed by GitHub
parent cae4927fbf
commit 1417345f5f
11 changed files with 796 additions and 42 deletions
@@ -219,6 +219,15 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
f"speculative_algorithm == EAGLE, got {cfg.speculative_algorithm}."
)
if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get():
if (
cfg.speculative_algorithm not in ("EAGLE", "EAGLE3")
or cfg.enable_multi_layer_eagle
):
raise ValueError(
"DP spec/prefill coordination requires single-layer EAGLE or EAGLE3"
)
if cfg.speculative_adaptive:
_maybe_disable_adaptive(server_args)
if cfg.speculative_adaptive:
+1
View File
@@ -1339,6 +1339,7 @@ class Envs:
# (EAGLE/MTP). Off by default; see the PP+spec RFC for constraints
# (non-overlap schedule, no DP attention).
SGLANG_ENABLE_PP_SPEC = EnvBool(False)
SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION = EnvBool(False)
# Capture the per-replay attention-metadata prep (init_forward_metadata_out_graph)
# into a small CUDA graph, collapsing its host dispatch cost to one launch.
# Experimental; auto-falls back to eager if the backend's prep is not capturable.
@@ -2446,6 +2446,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
forward_mode: ForwardMode = None
global_forward_mode: Optional[ForwardMode] = None
# Full-DP metadata from the existing scheduler gather.
dp_spec_prefill_coordination_metadata: Optional[tuple] = None
dp_spec_prefill_coordination_applied: bool = False
# For DP attention
is_extend_in_batch: bool = False
can_run_decode_cuda_graph: bool = False
@@ -3791,6 +3795,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
decoding_reqs=self.decoding_reqs,
spec_algorithm=self.spec_algorithm,
spec_info=self.spec_info,
dp_spec_prefill_coordination_metadata=self.dp_spec_prefill_coordination_metadata,
dp_spec_prefill_coordination_applied=self.dp_spec_prefill_coordination_applied,
global_num_tokens=self.global_num_tokens,
global_num_tokens_for_logprob=self.global_num_tokens_for_logprob,
can_run_decode_cuda_graph=self.can_run_decode_cuda_graph,
+1
View File
@@ -3713,6 +3713,7 @@ class Scheduler(
need_mlp_sync
and not self.spec_algorithm.is_none()
and not get_spec().speculative_skip_dp_mlp_sync
and not envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get()
):
# NOTE: This branch makes sure prefill and decode batches will not be mixed when spec and dp-attn is enabled.
# Before merging the new batch into running batch:
@@ -235,6 +235,15 @@ def _update_gather_batch(
batch.global_num_tokens_for_logprob = (
mlp_sync_info.global_num_tokens_for_logprob
)
if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get():
# Fresh counts have not yet been adjusted by the coordination plan.
batch.dp_spec_prefill_coordination_applied = False
info = mlp_sync_info.tp0_info_cpu
batch.dp_spec_prefill_coordination_metadata = (
mlp_sync_info.global_num_tokens,
mlp_sync_info.global_num_tokens_for_logprob,
info[:, 3],
)
if not skip_global_metadata:
batch.is_extend_in_batch = mlp_sync_info.is_extend_in_batch
batch.tbo_split_seq_index = mlp_sync_info.tbo_split_seq_index
@@ -571,6 +580,9 @@ class SchedulerDPAttnAdapter:
extend view when a peer rank runs extend this step, so the step stays
mode-homogeneous and every rank replays the extend graphs instead of
all falling to eager."""
if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get():
# Keep verification rows in their native mode on heterogeneous steps.
return batch
if batch is None or not batch.forward_mode.is_decode():
return batch
# Global triggers from the gather. This rank's own eligibility (spec/
@@ -540,6 +540,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# For DP attention
is_extend_in_batch: bool = False
dp_spec_prefill_coordination_applied: bool = False
can_run_decode_cuda_graph: bool = False
can_run_dp_prefill_cuda_graph: bool = False
dp_prefill_cuda_graph_max_prefix_len: int = 0
@@ -815,11 +816,17 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
self, batch: ScheduleBatch, device: Union[str, torch.device]
) -> None:
"""Populate per-rank token counts for DP-attention MLP synchronization."""
self.dp_spec_prefill_coordination_applied = (
batch.dp_spec_prefill_coordination_applied
)
if batch.global_num_tokens is None:
return
assert batch.global_num_tokens_for_logprob is not None
if self.spec_info is not None:
if (
self.spec_info is not None
and not batch.dp_spec_prefill_coordination_applied
):
from sglang.srt.speculative.spec_info import spec_scale_global_num_tokens
global_num_tokens, global_num_tokens_for_logprob = (
@@ -1551,6 +1558,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
and not prefill_graph_tolerates_sum_len()
):
dp_padding_mode = DpPaddingMode.MAX_LEN
if self.dp_spec_prefill_coordination_applied:
# Preserve each rank's forward mode on coordinated steps.
dp_padding_mode = DpPaddingMode.SUM_LEN
self.dp_padding_mode = dp_padding_mode
if dp_padding_mode.is_max_len():
@@ -0,0 +1,53 @@
"""Plan token counts for prefill and speculative decoding across DP ranks."""
from typing import List
import msgspec
import torch
class DPSpecPrefillCoordinationPlan(msgspec.Struct, frozen=True):
# Counts from the scheduler: prefill token counts or decode request counts.
counts: List[int]
logprob_counts: List[int]
prefills: torch.Tensor
draft_width: int
verify_width: int
@property
def heterogeneous(self):
return any(p and n for p, n in zip(self.prefills, self.counts)) and any(
not p and n for p, n in zip(self.prefills, self.counts)
)
def phase_counts(self, phase):
if phase == "draft":
tokens = [
0 if p else n * self.draft_width
for p, n in zip(self.prefills, self.counts)
]
return tokens, tokens
if phase not in ("target", "draft_extend"):
raise ValueError("Unknown DP speculative phase")
return (
[
n if p else n * self.verify_width
for p, n in zip(self.prefills, self.counts)
],
[
n if p else n * self.verify_width
for p, n in zip(self.prefills, self.logprob_counts)
],
)
def apply(self, batch, phase, rank, *, local_only):
tokens, logprobs = self.phase_counts(phase)
# Local drafts and EP-only batches retain only this rank's counts.
if local_only:
tokens, logprobs = [tokens[rank]], [logprobs[rank]]
batch.global_num_tokens = tokens
batch.global_num_tokens_for_logprob = logprobs
batch.dp_spec_prefill_coordination_applied = True
batch.is_extend_in_batch = True
batch.can_run_decode_cuda_graph = False
batch.can_run_dp_prefill_cuda_graph = False
+158 -41
View File
@@ -73,6 +73,9 @@ from sglang.srt.speculative.adaptive_runtime_state import (
)
from sglang.srt.speculative.adaptive_spec_params import AdaptiveSpeculativeParams
from sglang.srt.speculative.base_spec_worker import BaseSpecWorker, EagleDraftWorkerBase
from sglang.srt.speculative.dp_spec_prefill_coordination import (
DPSpecPrefillCoordinationPlan,
)
from sglang.srt.speculative.draft_utils import DraftBackendFactory
from sglang.srt.speculative.eagle_draft_cuda_graph_runner import (
EAGLEDraftCudaGraphRunner,
@@ -1305,6 +1308,10 @@ class EAGLEWorkerV2(BaseSpecWorker):
get_spec().speculative_algorithm
)
self.enable_dp_spec_prefill_coordination = (
envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get()
)
# Only the last PP stage runs the draft; other EAGLEWorkerV2 instances
# return proxies so scheduler dispatch remains rank-uniform.
self._hosts_draft = get_parallel().pp_group.is_last_rank
@@ -1400,49 +1407,26 @@ class EAGLEWorkerV2(BaseSpecWorker):
pp_proxy_tensors: Optional[PPProxyTensors] = None,
):
if batch.forward_mode.is_extend() or batch.is_extend_in_batch:
# Target prefill
target_capture_mode = (
CaptureHiddenMode.NULL
if self.speculative_algorithm.is_standalone()
else CaptureHiddenMode.FULL
)
batch_output = self.target_worker.forward_batch_generation(
batch,
pp_proxy_tensors=pp_proxy_tensors,
capture_hidden_mode=target_capture_mode,
)
# Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens.
# Extend processed L prompt tokens; next verify iter expects same L.
batch_output.new_seq_lens = batch.seq_lens
# Publish before draft_extend so the fence is at target-end.
if on_publish is not None:
on_publish(batch_output.new_seq_lens)
# A rank that does not host the draft (prefill-side PP builds it only on
# the last stage) forwards the target's proxy tensors and stops here.
if self._draft_worker is None:
return batch_output
# Draft prefill
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft_extend"),
if not (
batch.is_extend_in_batch and self.enable_dp_spec_prefill_coordination
):
batch_output.next_draft_input = (
self.draft_worker._draft_extend_for_prefill(
batch,
batch_output.logits_output.hidden_states,
batch_output.next_token_ids,
batch_output.logits_output.mm_input_embeds,
)
return self._forward_prefill_batch(batch, on_publish, pp_proxy_tensors)
else:
if batch.dp_spec_prefill_coordination_metadata is None:
raise RuntimeError("Missing DP spec/prefill coordination metadata")
plan = DPSpecPrefillCoordinationPlan(
*batch.dp_spec_prefill_coordination_metadata,
draft_width=self.topk,
verify_width=self.speculative_num_draft_tokens,
)
return batch_output
if not plan.heterogeneous:
return self._forward_prefill_batch(
batch, on_publish, pp_proxy_tensors
)
else:
return self._forward_dp_spec_prefill_coordination(
batch, plan, on_publish, grammar_barrier, pp_proxy_tensors
)
else:
self.activate_step_by_batch(batch.seq_lens.shape[0])
@@ -1554,6 +1538,139 @@ class EAGLEWorkerV2(BaseSpecWorker):
return batch_output
def _forward_prefill_batch(
self, batch, on_publish=None, pp_proxy_tensors=None, coordination_plan=None
):
# Target prefill
target_capture_mode = (
CaptureHiddenMode.NULL
if self.speculative_algorithm.is_standalone()
else CaptureHiddenMode.FULL
)
batch_output = self.target_worker.forward_batch_generation(
batch,
pp_proxy_tensors=pp_proxy_tensors,
capture_hidden_mode=target_capture_mode,
)
# Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens.
# Extend processed L prompt tokens; next verify iter expects same L.
batch_output.new_seq_lens = batch.seq_lens
# Publish before draft_extend so the fence is at target-end.
if on_publish is not None:
on_publish(batch_output.new_seq_lens)
# A rank that does not host the draft (prefill-side PP builds it only on
# the last stage) forwards the target's proxy tensors and stops here.
if self._draft_worker is None:
return batch_output
if coordination_plan is not None:
coordination_plan.apply(
batch,
"draft_extend",
get_parallel().attn_dp_rank,
local_only=(
len(batch.global_num_tokens) == 1
or self.draft_worker.draft_owns_attention
),
)
# Draft prefill
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft_extend"),
):
batch_output.next_draft_input = self.draft_worker._draft_extend_for_prefill(
batch,
batch_output.logits_output.hidden_states,
batch_output.next_token_ids,
batch_output.logits_output.mm_input_embeds,
)
return batch_output
def _forward_dp_spec_prefill_coordination(
self, batch, plan, on_publish, grammar_barrier, pp_proxy_tensors
):
"""Run draft, target, and draft-extend with each rank's local mode."""
rank = get_parallel().attn_dp_rank
target_local_only = len(batch.global_num_tokens) == 1
draft_local_only = target_local_only or self.draft_worker.draft_owns_attention
is_prefill = batch.forward_mode.is_extend()
if is_prefill and batch.decoding_reqs:
raise RuntimeError("Local mixed prefill/verify is not implemented")
draft_batch = batch
if is_prefill:
draft_batch = ScheduleBatch.init_new(
[],
batch.req_to_token_pool,
batch.token_to_kv_pool_allocator,
batch.tree_cache,
batch.model_config,
batch.enable_overlap,
batch.spec_algorithm,
)
draft_batch.prepare_for_idle()
draft_batch.global_num_tokens = batch.global_num_tokens
draft_batch.global_num_tokens_for_logprob = (
batch.global_num_tokens_for_logprob
)
if draft_batch.spec_info is None:
hidden_size, hidden_dtype = get_draft_recurrent_hidden_state_spec(
self.draft_worker.draft_runner
)
draft_batch.spec_info = EagleDraftInput.create_idle_input(
device=self.device,
hidden_size=hidden_size,
dtype=hidden_dtype,
topk=self.topk,
capture_hidden_mode=CaptureHiddenMode.LAST,
)
plan.apply(draft_batch, "draft", rank, local_only=draft_local_only)
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft"),
):
verify_input = self.draft_worker.draft(draft_batch)
plan.apply(batch, "target", rank, local_only=target_local_only)
if is_prefill:
result = self._forward_prefill_batch(
batch, on_publish, pp_proxy_tensors, coordination_plan=plan
)
# Pin the temporary idle tensors through the overlap lifetime.
result.extra_keep_alive_refs = list(result.extra_keep_alive_refs or ()) + [
draft_batch
]
return result
batch.spec_info = verify_input
result = self.verify(batch, grammar_barrier=grammar_barrier)
if on_publish is not None:
on_publish(result.new_seq_lens)
plan.apply(batch, "draft_extend", rank, local_only=draft_local_only)
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft_extend"),
):
self.draft_worker._draft_extend_for_decode(batch, result)
return result
def _build_trivial_verify_input(self, batch: ScheduleBatch) -> EagleVerifyInput:
"""Build a 1-node EagleVerifyInput rooted at the previous bonus token.