[Experimental] Preserve speculative decoding during prefill across DP ranks (#40118)
This commit is contained in:
@@ -219,6 +219,15 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
|
||||
f"speculative_algorithm == EAGLE, got {cfg.speculative_algorithm}."
|
||||
)
|
||||
|
||||
if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get():
|
||||
if (
|
||||
cfg.speculative_algorithm not in ("EAGLE", "EAGLE3")
|
||||
or cfg.enable_multi_layer_eagle
|
||||
):
|
||||
raise ValueError(
|
||||
"DP spec/prefill coordination requires single-layer EAGLE or EAGLE3"
|
||||
)
|
||||
|
||||
if cfg.speculative_adaptive:
|
||||
_maybe_disable_adaptive(server_args)
|
||||
if cfg.speculative_adaptive:
|
||||
|
||||
@@ -1339,6 +1339,7 @@ class Envs:
|
||||
# (EAGLE/MTP). Off by default; see the PP+spec RFC for constraints
|
||||
# (non-overlap schedule, no DP attention).
|
||||
SGLANG_ENABLE_PP_SPEC = EnvBool(False)
|
||||
SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION = EnvBool(False)
|
||||
# Capture the per-replay attention-metadata prep (init_forward_metadata_out_graph)
|
||||
# into a small CUDA graph, collapsing its host dispatch cost to one launch.
|
||||
# Experimental; auto-falls back to eager if the backend's prep is not capturable.
|
||||
|
||||
@@ -2446,6 +2446,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
forward_mode: ForwardMode = None
|
||||
global_forward_mode: Optional[ForwardMode] = None
|
||||
|
||||
# Full-DP metadata from the existing scheduler gather.
|
||||
dp_spec_prefill_coordination_metadata: Optional[tuple] = None
|
||||
dp_spec_prefill_coordination_applied: bool = False
|
||||
|
||||
# For DP attention
|
||||
is_extend_in_batch: bool = False
|
||||
can_run_decode_cuda_graph: bool = False
|
||||
@@ -3791,6 +3795,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
decoding_reqs=self.decoding_reqs,
|
||||
spec_algorithm=self.spec_algorithm,
|
||||
spec_info=self.spec_info,
|
||||
dp_spec_prefill_coordination_metadata=self.dp_spec_prefill_coordination_metadata,
|
||||
dp_spec_prefill_coordination_applied=self.dp_spec_prefill_coordination_applied,
|
||||
global_num_tokens=self.global_num_tokens,
|
||||
global_num_tokens_for_logprob=self.global_num_tokens_for_logprob,
|
||||
can_run_decode_cuda_graph=self.can_run_decode_cuda_graph,
|
||||
|
||||
@@ -3713,6 +3713,7 @@ class Scheduler(
|
||||
need_mlp_sync
|
||||
and not self.spec_algorithm.is_none()
|
||||
and not get_spec().speculative_skip_dp_mlp_sync
|
||||
and not envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get()
|
||||
):
|
||||
# NOTE: This branch makes sure prefill and decode batches will not be mixed when spec and dp-attn is enabled.
|
||||
# Before merging the new batch into running batch:
|
||||
|
||||
@@ -235,6 +235,15 @@ def _update_gather_batch(
|
||||
batch.global_num_tokens_for_logprob = (
|
||||
mlp_sync_info.global_num_tokens_for_logprob
|
||||
)
|
||||
if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get():
|
||||
# Fresh counts have not yet been adjusted by the coordination plan.
|
||||
batch.dp_spec_prefill_coordination_applied = False
|
||||
info = mlp_sync_info.tp0_info_cpu
|
||||
batch.dp_spec_prefill_coordination_metadata = (
|
||||
mlp_sync_info.global_num_tokens,
|
||||
mlp_sync_info.global_num_tokens_for_logprob,
|
||||
info[:, 3],
|
||||
)
|
||||
if not skip_global_metadata:
|
||||
batch.is_extend_in_batch = mlp_sync_info.is_extend_in_batch
|
||||
batch.tbo_split_seq_index = mlp_sync_info.tbo_split_seq_index
|
||||
@@ -571,6 +580,9 @@ class SchedulerDPAttnAdapter:
|
||||
extend view when a peer rank runs extend this step, so the step stays
|
||||
mode-homogeneous and every rank replays the extend graphs instead of
|
||||
all falling to eager."""
|
||||
if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get():
|
||||
# Keep verification rows in their native mode on heterogeneous steps.
|
||||
return batch
|
||||
if batch is None or not batch.forward_mode.is_decode():
|
||||
return batch
|
||||
# Global triggers from the gather. This rank's own eligibility (spec/
|
||||
|
||||
@@ -540,6 +540,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
|
||||
# For DP attention
|
||||
is_extend_in_batch: bool = False
|
||||
dp_spec_prefill_coordination_applied: bool = False
|
||||
can_run_decode_cuda_graph: bool = False
|
||||
can_run_dp_prefill_cuda_graph: bool = False
|
||||
dp_prefill_cuda_graph_max_prefix_len: int = 0
|
||||
@@ -815,11 +816,17 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
self, batch: ScheduleBatch, device: Union[str, torch.device]
|
||||
) -> None:
|
||||
"""Populate per-rank token counts for DP-attention MLP synchronization."""
|
||||
self.dp_spec_prefill_coordination_applied = (
|
||||
batch.dp_spec_prefill_coordination_applied
|
||||
)
|
||||
if batch.global_num_tokens is None:
|
||||
return
|
||||
|
||||
assert batch.global_num_tokens_for_logprob is not None
|
||||
if self.spec_info is not None:
|
||||
if (
|
||||
self.spec_info is not None
|
||||
and not batch.dp_spec_prefill_coordination_applied
|
||||
):
|
||||
from sglang.srt.speculative.spec_info import spec_scale_global_num_tokens
|
||||
|
||||
global_num_tokens, global_num_tokens_for_logprob = (
|
||||
@@ -1551,6 +1558,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
and not prefill_graph_tolerates_sum_len()
|
||||
):
|
||||
dp_padding_mode = DpPaddingMode.MAX_LEN
|
||||
if self.dp_spec_prefill_coordination_applied:
|
||||
# Preserve each rank's forward mode on coordinated steps.
|
||||
dp_padding_mode = DpPaddingMode.SUM_LEN
|
||||
self.dp_padding_mode = dp_padding_mode
|
||||
|
||||
if dp_padding_mode.is_max_len():
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Plan token counts for prefill and speculative decoding across DP ranks."""
|
||||
|
||||
from typing import List
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
|
||||
class DPSpecPrefillCoordinationPlan(msgspec.Struct, frozen=True):
|
||||
# Counts from the scheduler: prefill token counts or decode request counts.
|
||||
counts: List[int]
|
||||
logprob_counts: List[int]
|
||||
prefills: torch.Tensor
|
||||
draft_width: int
|
||||
verify_width: int
|
||||
|
||||
@property
|
||||
def heterogeneous(self):
|
||||
return any(p and n for p, n in zip(self.prefills, self.counts)) and any(
|
||||
not p and n for p, n in zip(self.prefills, self.counts)
|
||||
)
|
||||
|
||||
def phase_counts(self, phase):
|
||||
if phase == "draft":
|
||||
tokens = [
|
||||
0 if p else n * self.draft_width
|
||||
for p, n in zip(self.prefills, self.counts)
|
||||
]
|
||||
return tokens, tokens
|
||||
if phase not in ("target", "draft_extend"):
|
||||
raise ValueError("Unknown DP speculative phase")
|
||||
return (
|
||||
[
|
||||
n if p else n * self.verify_width
|
||||
for p, n in zip(self.prefills, self.counts)
|
||||
],
|
||||
[
|
||||
n if p else n * self.verify_width
|
||||
for p, n in zip(self.prefills, self.logprob_counts)
|
||||
],
|
||||
)
|
||||
|
||||
def apply(self, batch, phase, rank, *, local_only):
|
||||
tokens, logprobs = self.phase_counts(phase)
|
||||
# Local drafts and EP-only batches retain only this rank's counts.
|
||||
if local_only:
|
||||
tokens, logprobs = [tokens[rank]], [logprobs[rank]]
|
||||
batch.global_num_tokens = tokens
|
||||
batch.global_num_tokens_for_logprob = logprobs
|
||||
batch.dp_spec_prefill_coordination_applied = True
|
||||
batch.is_extend_in_batch = True
|
||||
batch.can_run_decode_cuda_graph = False
|
||||
batch.can_run_dp_prefill_cuda_graph = False
|
||||
@@ -73,6 +73,9 @@ from sglang.srt.speculative.adaptive_runtime_state import (
|
||||
)
|
||||
from sglang.srt.speculative.adaptive_spec_params import AdaptiveSpeculativeParams
|
||||
from sglang.srt.speculative.base_spec_worker import BaseSpecWorker, EagleDraftWorkerBase
|
||||
from sglang.srt.speculative.dp_spec_prefill_coordination import (
|
||||
DPSpecPrefillCoordinationPlan,
|
||||
)
|
||||
from sglang.srt.speculative.draft_utils import DraftBackendFactory
|
||||
from sglang.srt.speculative.eagle_draft_cuda_graph_runner import (
|
||||
EAGLEDraftCudaGraphRunner,
|
||||
@@ -1305,6 +1308,10 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
get_spec().speculative_algorithm
|
||||
)
|
||||
|
||||
self.enable_dp_spec_prefill_coordination = (
|
||||
envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get()
|
||||
)
|
||||
|
||||
# Only the last PP stage runs the draft; other EAGLEWorkerV2 instances
|
||||
# return proxies so scheduler dispatch remains rank-uniform.
|
||||
self._hosts_draft = get_parallel().pp_group.is_last_rank
|
||||
@@ -1400,49 +1407,26 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
):
|
||||
if batch.forward_mode.is_extend() or batch.is_extend_in_batch:
|
||||
# Target prefill
|
||||
target_capture_mode = (
|
||||
CaptureHiddenMode.NULL
|
||||
if self.speculative_algorithm.is_standalone()
|
||||
else CaptureHiddenMode.FULL
|
||||
)
|
||||
batch_output = self.target_worker.forward_batch_generation(
|
||||
batch,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
capture_hidden_mode=target_capture_mode,
|
||||
)
|
||||
|
||||
# Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens.
|
||||
# Extend processed L prompt tokens; next verify iter expects same L.
|
||||
batch_output.new_seq_lens = batch.seq_lens
|
||||
# Publish before draft_extend so the fence is at target-end.
|
||||
if on_publish is not None:
|
||||
on_publish(batch_output.new_seq_lens)
|
||||
|
||||
# A rank that does not host the draft (prefill-side PP builds it only on
|
||||
# the last stage) forwards the target's proxy tensors and stops here.
|
||||
if self._draft_worker is None:
|
||||
return batch_output
|
||||
|
||||
# Draft prefill
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft_extend"),
|
||||
if not (
|
||||
batch.is_extend_in_batch and self.enable_dp_spec_prefill_coordination
|
||||
):
|
||||
batch_output.next_draft_input = (
|
||||
self.draft_worker._draft_extend_for_prefill(
|
||||
batch,
|
||||
batch_output.logits_output.hidden_states,
|
||||
batch_output.next_token_ids,
|
||||
batch_output.logits_output.mm_input_embeds,
|
||||
)
|
||||
return self._forward_prefill_batch(batch, on_publish, pp_proxy_tensors)
|
||||
else:
|
||||
if batch.dp_spec_prefill_coordination_metadata is None:
|
||||
raise RuntimeError("Missing DP spec/prefill coordination metadata")
|
||||
plan = DPSpecPrefillCoordinationPlan(
|
||||
*batch.dp_spec_prefill_coordination_metadata,
|
||||
draft_width=self.topk,
|
||||
verify_width=self.speculative_num_draft_tokens,
|
||||
)
|
||||
return batch_output
|
||||
if not plan.heterogeneous:
|
||||
return self._forward_prefill_batch(
|
||||
batch, on_publish, pp_proxy_tensors
|
||||
)
|
||||
else:
|
||||
return self._forward_dp_spec_prefill_coordination(
|
||||
batch, plan, on_publish, grammar_barrier, pp_proxy_tensors
|
||||
)
|
||||
else:
|
||||
self.activate_step_by_batch(batch.seq_lens.shape[0])
|
||||
|
||||
@@ -1554,6 +1538,139 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
|
||||
return batch_output
|
||||
|
||||
def _forward_prefill_batch(
|
||||
self, batch, on_publish=None, pp_proxy_tensors=None, coordination_plan=None
|
||||
):
|
||||
# Target prefill
|
||||
target_capture_mode = (
|
||||
CaptureHiddenMode.NULL
|
||||
if self.speculative_algorithm.is_standalone()
|
||||
else CaptureHiddenMode.FULL
|
||||
)
|
||||
batch_output = self.target_worker.forward_batch_generation(
|
||||
batch,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
capture_hidden_mode=target_capture_mode,
|
||||
)
|
||||
|
||||
# Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens.
|
||||
# Extend processed L prompt tokens; next verify iter expects same L.
|
||||
batch_output.new_seq_lens = batch.seq_lens
|
||||
# Publish before draft_extend so the fence is at target-end.
|
||||
if on_publish is not None:
|
||||
on_publish(batch_output.new_seq_lens)
|
||||
|
||||
# A rank that does not host the draft (prefill-side PP builds it only on
|
||||
# the last stage) forwards the target's proxy tensors and stops here.
|
||||
if self._draft_worker is None:
|
||||
return batch_output
|
||||
|
||||
if coordination_plan is not None:
|
||||
coordination_plan.apply(
|
||||
batch,
|
||||
"draft_extend",
|
||||
get_parallel().attn_dp_rank,
|
||||
local_only=(
|
||||
len(batch.global_num_tokens) == 1
|
||||
or self.draft_worker.draft_owns_attention
|
||||
),
|
||||
)
|
||||
|
||||
# Draft prefill
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft_extend"),
|
||||
):
|
||||
batch_output.next_draft_input = self.draft_worker._draft_extend_for_prefill(
|
||||
batch,
|
||||
batch_output.logits_output.hidden_states,
|
||||
batch_output.next_token_ids,
|
||||
batch_output.logits_output.mm_input_embeds,
|
||||
)
|
||||
return batch_output
|
||||
|
||||
def _forward_dp_spec_prefill_coordination(
|
||||
self, batch, plan, on_publish, grammar_barrier, pp_proxy_tensors
|
||||
):
|
||||
"""Run draft, target, and draft-extend with each rank's local mode."""
|
||||
rank = get_parallel().attn_dp_rank
|
||||
target_local_only = len(batch.global_num_tokens) == 1
|
||||
draft_local_only = target_local_only or self.draft_worker.draft_owns_attention
|
||||
is_prefill = batch.forward_mode.is_extend()
|
||||
if is_prefill and batch.decoding_reqs:
|
||||
raise RuntimeError("Local mixed prefill/verify is not implemented")
|
||||
draft_batch = batch
|
||||
if is_prefill:
|
||||
draft_batch = ScheduleBatch.init_new(
|
||||
[],
|
||||
batch.req_to_token_pool,
|
||||
batch.token_to_kv_pool_allocator,
|
||||
batch.tree_cache,
|
||||
batch.model_config,
|
||||
batch.enable_overlap,
|
||||
batch.spec_algorithm,
|
||||
)
|
||||
draft_batch.prepare_for_idle()
|
||||
draft_batch.global_num_tokens = batch.global_num_tokens
|
||||
draft_batch.global_num_tokens_for_logprob = (
|
||||
batch.global_num_tokens_for_logprob
|
||||
)
|
||||
if draft_batch.spec_info is None:
|
||||
hidden_size, hidden_dtype = get_draft_recurrent_hidden_state_spec(
|
||||
self.draft_worker.draft_runner
|
||||
)
|
||||
draft_batch.spec_info = EagleDraftInput.create_idle_input(
|
||||
device=self.device,
|
||||
hidden_size=hidden_size,
|
||||
dtype=hidden_dtype,
|
||||
topk=self.topk,
|
||||
capture_hidden_mode=CaptureHiddenMode.LAST,
|
||||
)
|
||||
plan.apply(draft_batch, "draft", rank, local_only=draft_local_only)
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft"),
|
||||
):
|
||||
verify_input = self.draft_worker.draft(draft_batch)
|
||||
|
||||
plan.apply(batch, "target", rank, local_only=target_local_only)
|
||||
if is_prefill:
|
||||
result = self._forward_prefill_batch(
|
||||
batch, on_publish, pp_proxy_tensors, coordination_plan=plan
|
||||
)
|
||||
# Pin the temporary idle tensors through the overlap lifetime.
|
||||
result.extra_keep_alive_refs = list(result.extra_keep_alive_refs or ()) + [
|
||||
draft_batch
|
||||
]
|
||||
return result
|
||||
|
||||
batch.spec_info = verify_input
|
||||
result = self.verify(batch, grammar_barrier=grammar_barrier)
|
||||
if on_publish is not None:
|
||||
on_publish(result.new_seq_lens)
|
||||
plan.apply(batch, "draft_extend", rank, local_only=draft_local_only)
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft_extend"),
|
||||
):
|
||||
self.draft_worker._draft_extend_for_decode(batch, result)
|
||||
return result
|
||||
|
||||
def _build_trivial_verify_input(self, batch: ScheduleBatch) -> EagleVerifyInput:
|
||||
"""Build a 1-node EagleVerifyInput rooted at the previous bonus token.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user