[Spec/Prefill Coordination] Dspark support (#41623)
This commit is contained in:
@@ -221,11 +221,11 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
|
||||
|
||||
if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get():
|
||||
if (
|
||||
cfg.speculative_algorithm not in ("EAGLE", "EAGLE3")
|
||||
cfg.speculative_algorithm not in ("EAGLE", "EAGLE3", "DSPARK")
|
||||
or cfg.enable_multi_layer_eagle
|
||||
):
|
||||
raise ValueError(
|
||||
"DP spec/prefill coordination requires single-layer EAGLE or EAGLE3"
|
||||
"DP spec/prefill coordination requires single-layer EAGLE, EAGLE3 or DSPARK"
|
||||
)
|
||||
|
||||
if cfg.speculative_adaptive:
|
||||
@@ -598,7 +598,10 @@ def _handle_dspark(server_args: ServerArgs) -> None:
|
||||
"(built-in TP MoE), 'megamoe', or 'mori', got "
|
||||
f"{cfg.moe_a2a_backend!r}."
|
||||
)
|
||||
if not _is_npu and cfg.moe_a2a_backend != "none":
|
||||
if not _is_npu and (
|
||||
cfg.moe_a2a_backend != "none"
|
||||
or envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get()
|
||||
):
|
||||
from sglang.srt.speculative.ragged_verify import (
|
||||
RaggedVerifyMode,
|
||||
read_ragged_verify_mode,
|
||||
@@ -606,8 +609,7 @@ def _handle_dspark(server_args: ServerArgs) -> None:
|
||||
|
||||
if read_ragged_verify_mode() is not RaggedVerifyMode.STATIC:
|
||||
raise ValueError(
|
||||
"DSpark with dp attention + "
|
||||
f"moe_a2a_backend={cfg.moe_a2a_backend!r} requires "
|
||||
"DSpark DP MoE or prefill coordination requires "
|
||||
"SGLANG_RAGGED_VERIFY_MODE=static."
|
||||
)
|
||||
if cfg.attn_cp_size > 1:
|
||||
|
||||
@@ -478,6 +478,10 @@ class DraftBlockProposer:
|
||||
# The dense DSpark draft still reuses the target batch's graph tier.
|
||||
# Set graph eligibility before the DP-MoE-only metadata early return.
|
||||
forward_batch.can_run_decode_cuda_graph = batch.can_run_decode_cuda_graph
|
||||
forward_batch.is_extend_in_batch = batch.is_extend_in_batch
|
||||
forward_batch.dp_spec_prefill_coordination_applied = (
|
||||
batch.dp_spec_prefill_coordination_applied
|
||||
)
|
||||
device = self.draft_model_runner.device
|
||||
num_tokens = forward_batch.input_ids.numel()
|
||||
if self._num_token_non_padded is not None:
|
||||
@@ -490,11 +494,15 @@ class DraftBlockProposer:
|
||||
# them separate from global_num_tokens_cpu below, which is scaled into
|
||||
# draft-token units for DP/MoE synchronization.
|
||||
forward_batch.original_global_num_tokens_cpu = batch.global_num_tokens
|
||||
gnt, gnt_logprob = spec_scale_global_num_tokens(
|
||||
self._draft_block_spec_info,
|
||||
batch.global_num_tokens,
|
||||
batch.global_num_tokens_for_logprob,
|
||||
)
|
||||
if batch.dp_spec_prefill_coordination_applied:
|
||||
gnt = batch.global_num_tokens
|
||||
gnt_logprob = batch.global_num_tokens_for_logprob
|
||||
else:
|
||||
gnt, gnt_logprob = spec_scale_global_num_tokens(
|
||||
self._draft_block_spec_info,
|
||||
batch.global_num_tokens,
|
||||
batch.global_num_tokens_for_logprob,
|
||||
)
|
||||
forward_batch.original_global_num_tokens_cpu = batch.global_num_tokens
|
||||
num_tokens = forward_batch.input_ids.numel()
|
||||
num_token_non_padded = _make_num_token_non_padded(num_tokens, device)
|
||||
|
||||
@@ -32,6 +32,9 @@ from sglang.srt.runtime_context import (
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.speculative.base_spec_worker import BaseSpecWorker
|
||||
from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2
|
||||
from sglang.srt.speculative.dp_spec_prefill_coordination import (
|
||||
DPSpecPrefillCoordinationPlan,
|
||||
)
|
||||
from sglang.srt.speculative.draft_worker_common import (
|
||||
build_block_pos_offsets,
|
||||
build_draft_tp_worker,
|
||||
@@ -151,6 +154,9 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
self.page_size = get_schedule().page_size
|
||||
self.device = target_worker.device
|
||||
self._draft_worker = None
|
||||
self.enable_dp_spec_prefill_coordination = (
|
||||
envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get()
|
||||
)
|
||||
self._hosts_draft = get_parallel().pp_group.is_last_rank
|
||||
if not self._hosts_draft:
|
||||
return
|
||||
@@ -571,12 +577,46 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
on_publish(batch_output.new_seq_lens)
|
||||
return batch_output
|
||||
if batch.forward_mode.is_extend() or batch.is_extend_in_batch:
|
||||
if batch.is_extend_in_batch and self.enable_dp_spec_prefill_coordination:
|
||||
plan = DPSpecPrefillCoordinationPlan(
|
||||
*batch.dp_spec_prefill_coordination_metadata,
|
||||
draft_width=self._proposer.query_token_num,
|
||||
verify_width=self.verify_num_draft_tokens,
|
||||
)
|
||||
if plan.heterogeneous:
|
||||
return self._forward_dp_spec_prefill_coordination(
|
||||
batch, plan, on_publish, grammar_barrier, pp_proxy_tensors
|
||||
)
|
||||
self._verify_planner.note_non_decode_step()
|
||||
self._observers.note_prefill_step()
|
||||
return self._forward_prefill(batch, on_publish, pp_proxy_tensors)
|
||||
|
||||
return self._forward_decode(batch, on_publish, grammar_barrier)
|
||||
|
||||
def _forward_dp_spec_prefill_coordination(
|
||||
self, batch, plan, on_publish, grammar_barrier, pp_proxy_tensors
|
||||
):
|
||||
"""Pair local prefill with peer verification after the shared draft pass."""
|
||||
if not batch.forward_mode.is_extend():
|
||||
return self._forward_decode(
|
||||
batch, on_publish, grammar_barrier, coordination_plan=plan
|
||||
)
|
||||
|
||||
rank = get_parallel().attn_dp_rank
|
||||
target_local_only = len(batch.global_num_tokens) == 1
|
||||
plan.apply(
|
||||
batch,
|
||||
"draft",
|
||||
rank,
|
||||
local_only=target_local_only or self._draft_dp_context_enabled,
|
||||
)
|
||||
with self._draft_context():
|
||||
self._proposer.run_idle_participation(batch)
|
||||
plan.apply(batch, "target", rank, local_only=target_local_only)
|
||||
self._verify_planner.note_non_decode_step()
|
||||
self._observers.note_prefill_step()
|
||||
return self._forward_prefill(batch, on_publish, pp_proxy_tensors)
|
||||
|
||||
def _forward_prefill(
|
||||
self,
|
||||
batch: ScheduleBatch,
|
||||
@@ -732,7 +772,11 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
)
|
||||
|
||||
def _forward_decode(
|
||||
self, batch: ScheduleBatch, on_publish, grammar_barrier=None
|
||||
self,
|
||||
batch: ScheduleBatch,
|
||||
on_publish,
|
||||
grammar_barrier=None,
|
||||
coordination_plan=None,
|
||||
) -> GenerationBatchResult:
|
||||
if batch.spec_info is None:
|
||||
batch.spec_info = DFlashDraftInputV2.create_idle_input(device=self.device)
|
||||
@@ -742,11 +786,32 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
"DSpark spec-v2 expected DFlashDraftInputV2 state on the running batch."
|
||||
)
|
||||
|
||||
global_num_reqs = (
|
||||
max(batch.global_num_tokens)
|
||||
if self._draft_is_moe
|
||||
and get_parallel().enable_dp_attention
|
||||
and batch.global_num_tokens is not None
|
||||
else None
|
||||
)
|
||||
if coordination_plan is not None:
|
||||
rank = get_parallel().attn_dp_rank
|
||||
target_local_only = len(batch.global_num_tokens) == 1
|
||||
coordination_plan.apply(
|
||||
batch,
|
||||
"draft",
|
||||
rank,
|
||||
local_only=target_local_only or self._draft_dp_context_enabled,
|
||||
)
|
||||
|
||||
if batch.forward_mode.is_idle():
|
||||
self._observers.note_idle_decode_step()
|
||||
if get_parallel().enable_dp_attention:
|
||||
if self._draft_is_moe:
|
||||
self._proposer.run_idle_participation(batch)
|
||||
if coordination_plan is not None:
|
||||
coordination_plan.apply(
|
||||
batch, "target", rank, local_only=target_local_only
|
||||
)
|
||||
self._verify_executor.run_idle_participation(
|
||||
batch=batch, idle_layout=self._idle_verify_ragged_layout(batch)
|
||||
)
|
||||
@@ -785,6 +850,8 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
draft_block_ids = proposal.draft_block_ids
|
||||
draft_block = proposal.draft_block
|
||||
draft_tokens = draft_block.draft_tokens
|
||||
if coordination_plan is not None:
|
||||
coordination_plan.apply(batch, "target", rank, local_only=target_local_only)
|
||||
|
||||
confidence = proposal.confidence
|
||||
if confidence is None:
|
||||
@@ -802,13 +869,6 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
)
|
||||
|
||||
global_num_reqs = (
|
||||
max(batch.global_num_tokens)
|
||||
if self._draft_is_moe
|
||||
and get_parallel().enable_dp_attention
|
||||
and batch.global_num_tokens is not None
|
||||
else None
|
||||
)
|
||||
layout = self._verify_planner.schedule_layout(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
prefix_lens=prefix_lens,
|
||||
|
||||
Reference in New Issue
Block a user