[Spec/Prefill Coordination] Dspark support (#41623)

This commit is contained in:
Yuwei An
2026-09-29 16:08:01 -07:00
committed by GitHub
parent 211b1d9784
commit d3f8a8f4f5
6 changed files with 391 additions and 18 deletions
@@ -221,11 +221,11 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get():
if (
cfg.speculative_algorithm not in ("EAGLE", "EAGLE3")
cfg.speculative_algorithm not in ("EAGLE", "EAGLE3", "DSPARK")
or cfg.enable_multi_layer_eagle
):
raise ValueError(
"DP spec/prefill coordination requires single-layer EAGLE or EAGLE3"
"DP spec/prefill coordination requires single-layer EAGLE, EAGLE3 or DSPARK"
)
if cfg.speculative_adaptive:
@@ -598,7 +598,10 @@ def _handle_dspark(server_args: ServerArgs) -> None:
"(built-in TP MoE), 'megamoe', or 'mori', got "
f"{cfg.moe_a2a_backend!r}."
)
if not _is_npu and cfg.moe_a2a_backend != "none":
if not _is_npu and (
cfg.moe_a2a_backend != "none"
or envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get()
):
from sglang.srt.speculative.ragged_verify import (
RaggedVerifyMode,
read_ragged_verify_mode,
@@ -606,8 +609,7 @@ def _handle_dspark(server_args: ServerArgs) -> None:
if read_ragged_verify_mode() is not RaggedVerifyMode.STATIC:
raise ValueError(
"DSpark with dp attention + "
f"moe_a2a_backend={cfg.moe_a2a_backend!r} requires "
"DSpark DP MoE or prefill coordination requires "
"SGLANG_RAGGED_VERIFY_MODE=static."
)
if cfg.attn_cp_size > 1:
@@ -478,6 +478,10 @@ class DraftBlockProposer:
# The dense DSpark draft still reuses the target batch's graph tier.
# Set graph eligibility before the DP-MoE-only metadata early return.
forward_batch.can_run_decode_cuda_graph = batch.can_run_decode_cuda_graph
forward_batch.is_extend_in_batch = batch.is_extend_in_batch
forward_batch.dp_spec_prefill_coordination_applied = (
batch.dp_spec_prefill_coordination_applied
)
device = self.draft_model_runner.device
num_tokens = forward_batch.input_ids.numel()
if self._num_token_non_padded is not None:
@@ -490,11 +494,15 @@ class DraftBlockProposer:
# them separate from global_num_tokens_cpu below, which is scaled into
# draft-token units for DP/MoE synchronization.
forward_batch.original_global_num_tokens_cpu = batch.global_num_tokens
gnt, gnt_logprob = spec_scale_global_num_tokens(
self._draft_block_spec_info,
batch.global_num_tokens,
batch.global_num_tokens_for_logprob,
)
if batch.dp_spec_prefill_coordination_applied:
gnt = batch.global_num_tokens
gnt_logprob = batch.global_num_tokens_for_logprob
else:
gnt, gnt_logprob = spec_scale_global_num_tokens(
self._draft_block_spec_info,
batch.global_num_tokens,
batch.global_num_tokens_for_logprob,
)
forward_batch.original_global_num_tokens_cpu = batch.global_num_tokens
num_tokens = forward_batch.input_ids.numel()
num_token_non_padded = _make_num_token_non_padded(num_tokens, device)
@@ -32,6 +32,9 @@ from sglang.srt.runtime_context import (
from sglang.srt.server_args import ServerArgs
from sglang.srt.speculative.base_spec_worker import BaseSpecWorker
from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2
from sglang.srt.speculative.dp_spec_prefill_coordination import (
DPSpecPrefillCoordinationPlan,
)
from sglang.srt.speculative.draft_worker_common import (
build_block_pos_offsets,
build_draft_tp_worker,
@@ -151,6 +154,9 @@ class DSparkWorkerV2(BaseSpecWorker):
self.page_size = get_schedule().page_size
self.device = target_worker.device
self._draft_worker = None
self.enable_dp_spec_prefill_coordination = (
envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get()
)
self._hosts_draft = get_parallel().pp_group.is_last_rank
if not self._hosts_draft:
return
@@ -571,12 +577,46 @@ class DSparkWorkerV2(BaseSpecWorker):
on_publish(batch_output.new_seq_lens)
return batch_output
if batch.forward_mode.is_extend() or batch.is_extend_in_batch:
if batch.is_extend_in_batch and self.enable_dp_spec_prefill_coordination:
plan = DPSpecPrefillCoordinationPlan(
*batch.dp_spec_prefill_coordination_metadata,
draft_width=self._proposer.query_token_num,
verify_width=self.verify_num_draft_tokens,
)
if plan.heterogeneous:
return self._forward_dp_spec_prefill_coordination(
batch, plan, on_publish, grammar_barrier, pp_proxy_tensors
)
self._verify_planner.note_non_decode_step()
self._observers.note_prefill_step()
return self._forward_prefill(batch, on_publish, pp_proxy_tensors)
return self._forward_decode(batch, on_publish, grammar_barrier)
def _forward_dp_spec_prefill_coordination(
self, batch, plan, on_publish, grammar_barrier, pp_proxy_tensors
):
"""Pair local prefill with peer verification after the shared draft pass."""
if not batch.forward_mode.is_extend():
return self._forward_decode(
batch, on_publish, grammar_barrier, coordination_plan=plan
)
rank = get_parallel().attn_dp_rank
target_local_only = len(batch.global_num_tokens) == 1
plan.apply(
batch,
"draft",
rank,
local_only=target_local_only or self._draft_dp_context_enabled,
)
with self._draft_context():
self._proposer.run_idle_participation(batch)
plan.apply(batch, "target", rank, local_only=target_local_only)
self._verify_planner.note_non_decode_step()
self._observers.note_prefill_step()
return self._forward_prefill(batch, on_publish, pp_proxy_tensors)
def _forward_prefill(
self,
batch: ScheduleBatch,
@@ -732,7 +772,11 @@ class DSparkWorkerV2(BaseSpecWorker):
)
def _forward_decode(
self, batch: ScheduleBatch, on_publish, grammar_barrier=None
self,
batch: ScheduleBatch,
on_publish,
grammar_barrier=None,
coordination_plan=None,
) -> GenerationBatchResult:
if batch.spec_info is None:
batch.spec_info = DFlashDraftInputV2.create_idle_input(device=self.device)
@@ -742,11 +786,32 @@ class DSparkWorkerV2(BaseSpecWorker):
"DSpark spec-v2 expected DFlashDraftInputV2 state on the running batch."
)
global_num_reqs = (
max(batch.global_num_tokens)
if self._draft_is_moe
and get_parallel().enable_dp_attention
and batch.global_num_tokens is not None
else None
)
if coordination_plan is not None:
rank = get_parallel().attn_dp_rank
target_local_only = len(batch.global_num_tokens) == 1
coordination_plan.apply(
batch,
"draft",
rank,
local_only=target_local_only or self._draft_dp_context_enabled,
)
if batch.forward_mode.is_idle():
self._observers.note_idle_decode_step()
if get_parallel().enable_dp_attention:
if self._draft_is_moe:
self._proposer.run_idle_participation(batch)
if coordination_plan is not None:
coordination_plan.apply(
batch, "target", rank, local_only=target_local_only
)
self._verify_executor.run_idle_participation(
batch=batch, idle_layout=self._idle_verify_ragged_layout(batch)
)
@@ -785,6 +850,8 @@ class DSparkWorkerV2(BaseSpecWorker):
draft_block_ids = proposal.draft_block_ids
draft_block = proposal.draft_block
draft_tokens = draft_block.draft_tokens
if coordination_plan is not None:
coordination_plan.apply(batch, "target", rank, local_only=target_local_only)
confidence = proposal.confidence
if confidence is None:
@@ -802,13 +869,6 @@ class DSparkWorkerV2(BaseSpecWorker):
req_pool_indices=batch.req_pool_indices,
)
global_num_reqs = (
max(batch.global_num_tokens)
if self._draft_is_moe
and get_parallel().enable_dp_attention
and batch.global_num_tokens is not None
else None
)
layout = self._verify_planner.schedule_layout(
req_pool_indices=batch.req_pool_indices,
prefix_lens=prefix_lens,