diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py index 1da0defffc..09078eed9f 100644 --- a/python/sglang/srt/arg_groups/speculative_hook.py +++ b/python/sglang/srt/arg_groups/speculative_hook.py @@ -219,6 +219,15 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None: f"speculative_algorithm == EAGLE, got {cfg.speculative_algorithm}." ) + if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get(): + if ( + cfg.speculative_algorithm not in ("EAGLE", "EAGLE3") + or cfg.enable_multi_layer_eagle + ): + raise ValueError( + "DP spec/prefill coordination requires single-layer EAGLE or EAGLE3" + ) + if cfg.speculative_adaptive: _maybe_disable_adaptive(server_args) if cfg.speculative_adaptive: diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index a4ea55503c..adc7d08ed2 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1339,6 +1339,7 @@ class Envs: # (EAGLE/MTP). Off by default; see the PP+spec RFC for constraints # (non-overlap schedule, no DP attention). SGLANG_ENABLE_PP_SPEC = EnvBool(False) + SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION = EnvBool(False) # Capture the per-replay attention-metadata prep (init_forward_metadata_out_graph) # into a small CUDA graph, collapsing its host dispatch cost to one launch. # Experimental; auto-falls back to eager if the backend's prep is not capturable. diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 816ca3041e..105a44f970 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -2446,6 +2446,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): forward_mode: ForwardMode = None global_forward_mode: Optional[ForwardMode] = None + # Full-DP metadata from the existing scheduler gather. + dp_spec_prefill_coordination_metadata: Optional[tuple] = None + dp_spec_prefill_coordination_applied: bool = False + # For DP attention is_extend_in_batch: bool = False can_run_decode_cuda_graph: bool = False @@ -3791,6 +3795,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): decoding_reqs=self.decoding_reqs, spec_algorithm=self.spec_algorithm, spec_info=self.spec_info, + dp_spec_prefill_coordination_metadata=self.dp_spec_prefill_coordination_metadata, + dp_spec_prefill_coordination_applied=self.dp_spec_prefill_coordination_applied, global_num_tokens=self.global_num_tokens, global_num_tokens_for_logprob=self.global_num_tokens_for_logprob, can_run_decode_cuda_graph=self.can_run_decode_cuda_graph, diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 9adf9d63d5..2236dc138d 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -3713,6 +3713,7 @@ class Scheduler( need_mlp_sync and not self.spec_algorithm.is_none() and not get_spec().speculative_skip_dp_mlp_sync + and not envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get() ): # NOTE: This branch makes sure prefill and decode batches will not be mixed when spec and dp-attn is enabled. # Before merging the new batch into running batch: diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index 57b6c6e1c1..ffa1f3c1eb 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -235,6 +235,15 @@ def _update_gather_batch( batch.global_num_tokens_for_logprob = ( mlp_sync_info.global_num_tokens_for_logprob ) + if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get(): + # Fresh counts have not yet been adjusted by the coordination plan. + batch.dp_spec_prefill_coordination_applied = False + info = mlp_sync_info.tp0_info_cpu + batch.dp_spec_prefill_coordination_metadata = ( + mlp_sync_info.global_num_tokens, + mlp_sync_info.global_num_tokens_for_logprob, + info[:, 3], + ) if not skip_global_metadata: batch.is_extend_in_batch = mlp_sync_info.is_extend_in_batch batch.tbo_split_seq_index = mlp_sync_info.tbo_split_seq_index @@ -571,6 +580,9 @@ class SchedulerDPAttnAdapter: extend view when a peer rank runs extend this step, so the step stays mode-homogeneous and every rank replays the extend graphs instead of all falling to eager.""" + if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get(): + # Keep verification rows in their native mode on heterogeneous steps. + return batch if batch is None or not batch.forward_mode.is_decode(): return batch # Global triggers from the gather. This rank's own eligibility (spec/ diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 8ef752766d..1bd0830e24 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -540,6 +540,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # For DP attention is_extend_in_batch: bool = False + dp_spec_prefill_coordination_applied: bool = False can_run_decode_cuda_graph: bool = False can_run_dp_prefill_cuda_graph: bool = False dp_prefill_cuda_graph_max_prefix_len: int = 0 @@ -815,11 +816,17 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): self, batch: ScheduleBatch, device: Union[str, torch.device] ) -> None: """Populate per-rank token counts for DP-attention MLP synchronization.""" + self.dp_spec_prefill_coordination_applied = ( + batch.dp_spec_prefill_coordination_applied + ) if batch.global_num_tokens is None: return assert batch.global_num_tokens_for_logprob is not None - if self.spec_info is not None: + if ( + self.spec_info is not None + and not batch.dp_spec_prefill_coordination_applied + ): from sglang.srt.speculative.spec_info import spec_scale_global_num_tokens global_num_tokens, global_num_tokens_for_logprob = ( @@ -1551,6 +1558,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): and not prefill_graph_tolerates_sum_len() ): dp_padding_mode = DpPaddingMode.MAX_LEN + if self.dp_spec_prefill_coordination_applied: + # Preserve each rank's forward mode on coordinated steps. + dp_padding_mode = DpPaddingMode.SUM_LEN self.dp_padding_mode = dp_padding_mode if dp_padding_mode.is_max_len(): diff --git a/python/sglang/srt/speculative/dp_spec_prefill_coordination.py b/python/sglang/srt/speculative/dp_spec_prefill_coordination.py new file mode 100644 index 0000000000..cbd9a056ac --- /dev/null +++ b/python/sglang/srt/speculative/dp_spec_prefill_coordination.py @@ -0,0 +1,53 @@ +"""Plan token counts for prefill and speculative decoding across DP ranks.""" + +from typing import List + +import msgspec +import torch + + +class DPSpecPrefillCoordinationPlan(msgspec.Struct, frozen=True): + # Counts from the scheduler: prefill token counts or decode request counts. + counts: List[int] + logprob_counts: List[int] + prefills: torch.Tensor + draft_width: int + verify_width: int + + @property + def heterogeneous(self): + return any(p and n for p, n in zip(self.prefills, self.counts)) and any( + not p and n for p, n in zip(self.prefills, self.counts) + ) + + def phase_counts(self, phase): + if phase == "draft": + tokens = [ + 0 if p else n * self.draft_width + for p, n in zip(self.prefills, self.counts) + ] + return tokens, tokens + if phase not in ("target", "draft_extend"): + raise ValueError("Unknown DP speculative phase") + return ( + [ + n if p else n * self.verify_width + for p, n in zip(self.prefills, self.counts) + ], + [ + n if p else n * self.verify_width + for p, n in zip(self.prefills, self.logprob_counts) + ], + ) + + def apply(self, batch, phase, rank, *, local_only): + tokens, logprobs = self.phase_counts(phase) + # Local drafts and EP-only batches retain only this rank's counts. + if local_only: + tokens, logprobs = [tokens[rank]], [logprobs[rank]] + batch.global_num_tokens = tokens + batch.global_num_tokens_for_logprob = logprobs + batch.dp_spec_prefill_coordination_applied = True + batch.is_extend_in_batch = True + batch.can_run_decode_cuda_graph = False + batch.can_run_dp_prefill_cuda_graph = False diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index bbe5e4733a..e353f61700 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -73,6 +73,9 @@ from sglang.srt.speculative.adaptive_runtime_state import ( ) from sglang.srt.speculative.adaptive_spec_params import AdaptiveSpeculativeParams from sglang.srt.speculative.base_spec_worker import BaseSpecWorker, EagleDraftWorkerBase +from sglang.srt.speculative.dp_spec_prefill_coordination import ( + DPSpecPrefillCoordinationPlan, +) from sglang.srt.speculative.draft_utils import DraftBackendFactory from sglang.srt.speculative.eagle_draft_cuda_graph_runner import ( EAGLEDraftCudaGraphRunner, @@ -1305,6 +1308,10 @@ class EAGLEWorkerV2(BaseSpecWorker): get_spec().speculative_algorithm ) + self.enable_dp_spec_prefill_coordination = ( + envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get() + ) + # Only the last PP stage runs the draft; other EAGLEWorkerV2 instances # return proxies so scheduler dispatch remains rank-uniform. self._hosts_draft = get_parallel().pp_group.is_last_rank @@ -1400,49 +1407,26 @@ class EAGLEWorkerV2(BaseSpecWorker): pp_proxy_tensors: Optional[PPProxyTensors] = None, ): if batch.forward_mode.is_extend() or batch.is_extend_in_batch: - # Target prefill - target_capture_mode = ( - CaptureHiddenMode.NULL - if self.speculative_algorithm.is_standalone() - else CaptureHiddenMode.FULL - ) - batch_output = self.target_worker.forward_batch_generation( - batch, - pp_proxy_tensors=pp_proxy_tensors, - capture_hidden_mode=target_capture_mode, - ) - - # Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens. - # Extend processed L prompt tokens; next verify iter expects same L. - batch_output.new_seq_lens = batch.seq_lens - # Publish before draft_extend so the fence is at target-end. - if on_publish is not None: - on_publish(batch_output.new_seq_lens) - - # A rank that does not host the draft (prefill-side PP builds it only on - # the last stage) forwards the target's proxy tensors and stops here. - if self._draft_worker is None: - return batch_output - - # Draft prefill - with ( - self.draft_worker.draft_tp_context( - self.draft_worker.draft_runner.tp_group, - owns_attention=self.draft_worker.draft_owns_attention, - ), - speculative_moe_backend_context(), - speculative_moe_a2a_backend_context(), - spec_stage_span("draft_extend"), + if not ( + batch.is_extend_in_batch and self.enable_dp_spec_prefill_coordination ): - batch_output.next_draft_input = ( - self.draft_worker._draft_extend_for_prefill( - batch, - batch_output.logits_output.hidden_states, - batch_output.next_token_ids, - batch_output.logits_output.mm_input_embeds, - ) + return self._forward_prefill_batch(batch, on_publish, pp_proxy_tensors) + else: + if batch.dp_spec_prefill_coordination_metadata is None: + raise RuntimeError("Missing DP spec/prefill coordination metadata") + plan = DPSpecPrefillCoordinationPlan( + *batch.dp_spec_prefill_coordination_metadata, + draft_width=self.topk, + verify_width=self.speculative_num_draft_tokens, ) - return batch_output + if not plan.heterogeneous: + return self._forward_prefill_batch( + batch, on_publish, pp_proxy_tensors + ) + else: + return self._forward_dp_spec_prefill_coordination( + batch, plan, on_publish, grammar_barrier, pp_proxy_tensors + ) else: self.activate_step_by_batch(batch.seq_lens.shape[0]) @@ -1554,6 +1538,139 @@ class EAGLEWorkerV2(BaseSpecWorker): return batch_output + def _forward_prefill_batch( + self, batch, on_publish=None, pp_proxy_tensors=None, coordination_plan=None + ): + # Target prefill + target_capture_mode = ( + CaptureHiddenMode.NULL + if self.speculative_algorithm.is_standalone() + else CaptureHiddenMode.FULL + ) + batch_output = self.target_worker.forward_batch_generation( + batch, + pp_proxy_tensors=pp_proxy_tensors, + capture_hidden_mode=target_capture_mode, + ) + + # Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens. + # Extend processed L prompt tokens; next verify iter expects same L. + batch_output.new_seq_lens = batch.seq_lens + # Publish before draft_extend so the fence is at target-end. + if on_publish is not None: + on_publish(batch_output.new_seq_lens) + + # A rank that does not host the draft (prefill-side PP builds it only on + # the last stage) forwards the target's proxy tensors and stops here. + if self._draft_worker is None: + return batch_output + + if coordination_plan is not None: + coordination_plan.apply( + batch, + "draft_extend", + get_parallel().attn_dp_rank, + local_only=( + len(batch.global_num_tokens) == 1 + or self.draft_worker.draft_owns_attention + ), + ) + + # Draft prefill + with ( + self.draft_worker.draft_tp_context( + self.draft_worker.draft_runner.tp_group, + owns_attention=self.draft_worker.draft_owns_attention, + ), + speculative_moe_backend_context(), + speculative_moe_a2a_backend_context(), + spec_stage_span("draft_extend"), + ): + batch_output.next_draft_input = self.draft_worker._draft_extend_for_prefill( + batch, + batch_output.logits_output.hidden_states, + batch_output.next_token_ids, + batch_output.logits_output.mm_input_embeds, + ) + return batch_output + + def _forward_dp_spec_prefill_coordination( + self, batch, plan, on_publish, grammar_barrier, pp_proxy_tensors + ): + """Run draft, target, and draft-extend with each rank's local mode.""" + rank = get_parallel().attn_dp_rank + target_local_only = len(batch.global_num_tokens) == 1 + draft_local_only = target_local_only or self.draft_worker.draft_owns_attention + is_prefill = batch.forward_mode.is_extend() + if is_prefill and batch.decoding_reqs: + raise RuntimeError("Local mixed prefill/verify is not implemented") + draft_batch = batch + if is_prefill: + draft_batch = ScheduleBatch.init_new( + [], + batch.req_to_token_pool, + batch.token_to_kv_pool_allocator, + batch.tree_cache, + batch.model_config, + batch.enable_overlap, + batch.spec_algorithm, + ) + draft_batch.prepare_for_idle() + draft_batch.global_num_tokens = batch.global_num_tokens + draft_batch.global_num_tokens_for_logprob = ( + batch.global_num_tokens_for_logprob + ) + if draft_batch.spec_info is None: + hidden_size, hidden_dtype = get_draft_recurrent_hidden_state_spec( + self.draft_worker.draft_runner + ) + draft_batch.spec_info = EagleDraftInput.create_idle_input( + device=self.device, + hidden_size=hidden_size, + dtype=hidden_dtype, + topk=self.topk, + capture_hidden_mode=CaptureHiddenMode.LAST, + ) + plan.apply(draft_batch, "draft", rank, local_only=draft_local_only) + with ( + self.draft_worker.draft_tp_context( + self.draft_worker.draft_runner.tp_group, + owns_attention=self.draft_worker.draft_owns_attention, + ), + speculative_moe_backend_context(), + speculative_moe_a2a_backend_context(), + spec_stage_span("draft"), + ): + verify_input = self.draft_worker.draft(draft_batch) + + plan.apply(batch, "target", rank, local_only=target_local_only) + if is_prefill: + result = self._forward_prefill_batch( + batch, on_publish, pp_proxy_tensors, coordination_plan=plan + ) + # Pin the temporary idle tensors through the overlap lifetime. + result.extra_keep_alive_refs = list(result.extra_keep_alive_refs or ()) + [ + draft_batch + ] + return result + + batch.spec_info = verify_input + result = self.verify(batch, grammar_barrier=grammar_barrier) + if on_publish is not None: + on_publish(result.new_seq_lens) + plan.apply(batch, "draft_extend", rank, local_only=draft_local_only) + with ( + self.draft_worker.draft_tp_context( + self.draft_worker.draft_runner.tp_group, + owns_attention=self.draft_worker.draft_owns_attention, + ), + speculative_moe_backend_context(), + speculative_moe_a2a_backend_context(), + spec_stage_span("draft_extend"), + ): + self.draft_worker._draft_extend_for_decode(batch, result) + return result + def _build_trivial_verify_input(self, batch: ScheduleBatch) -> EagleVerifyInput: """Build a 1-node EagleVerifyInput rooted at the previous bonus token. diff --git a/test/registered/spec/eagle/test_dp_spec_prefill_coordination.py b/test/registered/spec/eagle/test_dp_spec_prefill_coordination.py new file mode 100644 index 0000000000..5bd05c26d8 --- /dev/null +++ b/test/registered/spec/eagle/test_dp_spec_prefill_coordination.py @@ -0,0 +1,114 @@ +import time +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor + +import requests + +from sglang.srt.environ import envs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ( + DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN, + DEFAULT_TARGET_MODEL_EAGLE_DP_ATTN, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + kill_process_tree, + popen_launch_server, +) + +register_cuda_ci(est_time=120, stage="base-c", runner_config="4-gpu-h100") + + +class TestDPSpecPrefillCoordination(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_TARGET_MODEL_EAGLE_DP_ATTN + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--speculative-algorithm", + "EAGLE3", + "--speculative-num-steps", + "6", + "--speculative-eagle-topk", + "10", + "--speculative-num-draft-tokens", + "32", + "--speculative-draft-model-path", + DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN, + "--tp-size", + "2", + "--dp-size", + "2", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--moe-dense-tp-size", + "1", + "--attention-backend", + "fa3", + "--mem-fraction-static", + "0.75", + "--cuda-graph-max-bs-decode", + "64", + ] + with ( + envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1), + envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.override(True), + ): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def _generate(self, text, rank, tokens=256): + response = requests.post( + self.base_url + "/generate", + json={ + "text": text, + "routed_dp_rank": rank, + "sampling_params": { + "temperature": 0, + "max_new_tokens": tokens, + "ignore_eos": True, + "sampling_seed": 42, + }, + }, + timeout=240, + ) + response.raise_for_status() + result = response.json() + self.assertEqual(result["meta_info"]["completion_tokens"], tokens) + output = result.get("output_ids") or result["text"] + self.assertTrue(output) + return output + + def test_decode_matches_isolated_output_during_prefill(self): + prompt = "Continue the positive integers separated by commas: 1, 2, 3," + expected = {rank: self._generate(prompt, rank) for rank in range(2)} + for decode_rank in range(2): + long_prompt = ( + f"Fresh context {uuid.uuid4().hex}:\n" + + "\n".join( + f"Entry {i}: apples are fruit; water is liquid; the sky appears blue." + for i in range(1536) + ) + + "\nSummarize these entries briefly." + ) + with self.subTest(decode_rank=decode_rank), ThreadPoolExecutor(2) as pool: + decode = pool.submit(self._generate, prompt, decode_rank) + time.sleep(0.5) + prefill = pool.submit(self._generate, long_prompt, 1 - decode_rank, 32) + self.assertEqual(decode.result(), expected[decode_rank]) + prefill.result() + self.assertEqual(self._generate(prompt, decode_rank), expected[decode_rank]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py b/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py index a7c9565e19..2a74025639 100644 --- a/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py +++ b/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py @@ -86,6 +86,7 @@ class TestMlpSyncPadUnpad(CustomTestCase): global_num_tokens=[2, 0, 3], global_num_tokens_for_logprob=[2, 0, 3], can_run_decode_cuda_graph=True, + dp_spec_prefill_coordination_applied=False, ) fb.init_mlp_sync_metadata(batch, torch.device("cpu")) diff --git a/test/registered/unit/spec/test_dp_spec_prefill_coordination.py b/test/registered/unit/spec/test_dp_spec_prefill_coordination.py new file mode 100644 index 0000000000..f70af17ee9 --- /dev/null +++ b/test/registered/unit/spec/test_dp_spec_prefill_coordination.py @@ -0,0 +1,430 @@ +import contextlib +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import torch + +from sglang.srt.environ import envs +from sglang.srt.layers.dp_attention import DpPaddingMode +from sglang.srt.managers.scheduler_components.dp_attn import _update_gather_batch +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.speculative.dp_spec_prefill_coordination import ( + DPSpecPrefillCoordinationPlan, +) +from sglang.srt.speculative.eagle_worker_v2 import EAGLEWorkerV2 +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +WORKER_MODULE = "sglang.srt.speculative.eagle_worker_v2" + + +def make_plan(): + return DPSpecPrefillCoordinationPlan( + counts=[4096, 32, 0, 512, 7, 1, 19, 0], + logprob_counts=[1, 32, 0, 2, 7, 1, 19, 0], + prefills=torch.tensor([1, 0, 0, 1, 0, 0, 0, 0]), + draft_width=1, + verify_width=4, + ) + + +def make_batch(counts, logprobs): + return SimpleNamespace( + global_num_tokens=list(counts), + global_num_tokens_for_logprob=list(logprobs), + dp_spec_prefill_coordination_applied=False, + is_extend_in_batch=False, + can_run_decode_cuda_graph=True, + can_run_dp_prefill_cuda_graph=True, + ) + + +class TestDPSpecPrefillCoordinationPlan(CustomTestCase): + def setUp(self): + self.plan = make_plan() + + def test_target_preserves_prefill_counts_and_scales_decode(self): + expected = ( + [4096, 128, 0, 512, 28, 4, 76, 0], + [1, 128, 0, 2, 28, 4, 76, 0], + ) + self.assertEqual(self.plan.phase_counts("target"), expected) + self.assertEqual(self.plan.phase_counts("draft_extend"), expected) + + def test_prefill_ranks_contribute_zero_draft_tokens(self): + self.assertEqual( + self.plan.phase_counts("draft"), + ([0, 32, 0, 0, 7, 1, 19, 0],) * 2, + ) + + def test_uniform_or_idle_steps_are_not_heterogeneous(self): + for counts, prefills in [ + ((0, 0), (False, False)), + ((4096, 0), (True, False)), + ((4096, 512), (True, True)), + ((32, 8), (False, False)), + ]: + with self.subTest(counts=counts, prefills=prefills): + self.assertFalse( + DPSpecPrefillCoordinationPlan( + list(counts), list(counts), torch.tensor(prefills), 1, 4 + ).heterogeneous + ) + self.assertTrue(self.plan.heterogeneous) + + def test_all_phases_disable_graphs_and_preserve_group_representation(self): + for phase in ("draft", "target", "draft_extend"): + tokens, logprobs = self.plan.phase_counts(phase) + for rank in range(8): + for local_only in (False, True): + with self.subTest(phase=phase, rank=rank, local_only=local_only): + batch = make_batch( + [self.plan.counts[rank]] + if local_only + else self.plan.counts, + [self.plan.logprob_counts[rank]] + if local_only + else self.plan.logprob_counts, + ) + self.plan.apply(batch, phase, rank, local_only=local_only) + self.assertEqual( + batch.global_num_tokens, + [tokens[rank]] if local_only else tokens, + ) + self.assertEqual( + batch.global_num_tokens_for_logprob, + [logprobs[rank]] if local_only else logprobs, + ) + self.assertTrue(batch.dp_spec_prefill_coordination_applied) + self.assertTrue(batch.is_extend_in_batch) + self.assertFalse(batch.can_run_decode_cuda_graph) + self.assertFalse(batch.can_run_dp_prefill_cuda_graph) + + def test_local_draft_restores_target_counts(self): + plan = DPSpecPrefillCoordinationPlan( + [4096, 24], [1, 24], torch.tensor([1, 0]), 10, 32 + ) + batch = make_batch(plan.counts, plan.logprob_counts) + plan.apply(batch, "draft", 1, local_only=True) + self.assertEqual(batch.global_num_tokens, [240]) + plan.apply(batch, "target", 1, local_only=False) + self.assertEqual(batch.global_num_tokens, [4096, 768]) + self.assertEqual(batch.global_num_tokens_for_logprob, [1, 768]) + plan.apply(batch, "draft_extend", 1, local_only=True) + self.assertEqual(batch.global_num_tokens, [768]) + + def test_invalid_phase_is_rejected(self): + with self.assertRaises(ValueError): + self.plan.phase_counts("unknown") + + def test_gather_resets_coordination_and_restores_pure_decode_flags(self): + for local_only in (False, True): + with self.subTest(local_only=local_only): + batch = make_batch([128], [128]) + batch.dp_spec_prefill_coordination_applied = True + info = SimpleNamespace( + num_tokens=32, + num_tokens_for_logprob=32, + global_num_tokens=[32, 7], + global_num_tokens_for_logprob=[32, 7], + tp0_info_cpu=torch.tensor([[32, 32, 1, 0], [7, 7, 1, 0]]), + is_extend_in_batch=False, + tbo_split_seq_index=None, + global_forward_mode=ForwardMode.DECODE, + can_run_decode_cuda_graph=True, + can_run_prefill_cuda_graph=False, + prefill_cuda_graph_max_prefix_len=0, + ) + with envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.override(True): + _update_gather_batch(batch, info, not local_only) + self.assertFalse(batch.dp_spec_prefill_coordination_applied) + self.assertEqual( + batch.global_num_tokens, [32] if local_only else [32, 7] + ) + counts, logprobs, prefills = batch.dp_spec_prefill_coordination_metadata + self.assertIs(counts, info.global_num_tokens) + self.assertIs(logprobs, info.global_num_tokens_for_logprob) + torch.testing.assert_close(prefills, info.tp0_info_cpu[:, 3]) + self.assertTrue(batch.can_run_decode_cuda_graph) + self.assertFalse(batch.is_extend_in_batch) + + def test_forward_metadata_scales_only_raw_counts(self): + for applied in (False, True): + with self.subTest(applied=applied): + batch = make_batch([4096, 128], [1, 128]) + batch.dp_spec_prefill_coordination_applied = applied + forward = object.__new__(ForwardBatch) + forward.spec_info = object() + with patch( + "sglang.srt.speculative.spec_info.spec_scale_global_num_tokens", + return_value=([16384, 512], [4, 512]), + ) as scale: + forward.init_mlp_sync_metadata(batch, "cpu") + self.assertEqual(forward.dp_spec_prefill_coordination_applied, applied) + self.assertEqual(scale.call_count, int(not applied)) + self.assertEqual( + forward.global_num_tokens_cpu, + [16384, 512] if not applied else [4096, 128], + ) + + def test_coordinated_padding_preserves_local_forward_modes(self): + module = "sglang.srt.model_executor.forward_batch_info" + for mode in ( + ForwardMode.DECODE, + ForwardMode.TARGET_VERIFY, + ForwardMode.DRAFT_EXTEND_V2, + ForwardMode.IDLE, + ): + for hybrid in (False, True): + with self.subTest(mode=mode, hybrid=hybrid): + idle = mode.is_idle() + width = 1 if mode.is_decode() or idle else 4 + tokens = 0 if idle else 2 * width + counts = [tokens, 8] if idle else [0, tokens] + batch = ForwardBatch( + forward_mode=mode, + batch_size=0 if idle else 2, + input_ids=torch.arange(tokens), + req_pool_indices=torch.arange(0 if idle else 2), + seq_lens=torch.tensor([] if idle else [5, 6]), + out_cache_loc=torch.arange(tokens), + seq_lens_sum=0 if idle else 11, + is_extend_in_batch=True, + dp_spec_prefill_coordination_applied=True, + global_num_tokens_cpu=counts, + global_num_tokens_for_logprob_cpu=counts, + global_num_tokens_gpu=torch.tensor(counts), + spec_info=SimpleNamespace( + num_tokens_per_req=width, + is_draft_input=lambda: True, + ), + ) + runner = MagicMock(enable_elastic_ep=False) + with ( + patch( + f"{module}.get_parallel", + return_value=SimpleNamespace(attn_tp_size=1), + ), + patch( + f"{module}.get_exec", + return_value=SimpleNamespace( + graph=SimpleNamespace( + cuda_graph_config=SimpleNamespace( + prefill=SimpleNamespace(bs=[]) + ) + ) + ), + ), + patch.object( + DpPaddingMode, + "get_dp_padding_mode", + return_value=DpPaddingMode.MAX_LEN, + ), + patch( + f"{module}.dp_gather_slot", return_value=0 if idle else 1 + ), + patch(f"{module}.set_dp_buffer_len"), + patch(f"{module}.set_is_extend_in_batch"), + patch( + f"{module}.mambaish_config", + return_value=object() if hybrid else None, + ), + patch(f"{module}._is_cpu", True), + patch.object(ForwardBatch, "_pad_inputs_to_size"), + patch( + "sglang.srt.batch_overlap.two_batch_overlap.TboForwardBatchPreparer.prepare" + ), + ): + batch.prepare_mlp_sync_batch(runner) + self.assertEqual(batch.dp_padding_mode, DpPaddingMode.SUM_LEN) + self.assertEqual(batch.forward_mode, mode) + self.assertEqual(batch.global_num_tokens_cpu, counts) + self.assertEqual(batch.batch_size, 0 if idle else 2) + + +class TestDPSpecPrefillCoordinationWorker(CustomTestCase): + def test_prefill_decode_and_idle_ranks_follow_the_same_phase_order(self): + plan = make_plan() + for rank in range(8): + with self.subTest(rank=rank): + events = [] + is_prefill = plan.prefills[rank] + batch = make_batch(plan.counts, plan.logprob_counts) + batch.forward_mode = ( + ForwardMode.EXTEND + if is_prefill + else ForwardMode.DECODE + if plan.counts[rank] + else ForwardMode.IDLE + ) + batch.decoding_reqs = None + batch.req_to_token_pool = batch.token_to_kv_pool_allocator = None + batch.tree_cache = batch.model_config = None + batch.enable_overlap = True + batch.spec_algorithm = SpeculativeAlgorithm.EAGLE + batch.spec_info = ( + None if is_prefill or not plan.counts[rank] else object() + ) + batch.seq_lens = torch.tensor([12]) + idle = make_batch(plan.counts, plan.logprob_counts) + idle.spec_info = None + idle.prepare_for_idle = MagicMock() + result = SimpleNamespace( + new_seq_lens=batch.seq_lens, + extra_keep_alive_refs=None, + logits_output=SimpleNamespace( + hidden_states=object(), mm_input_embeds=None + ), + next_token_ids=object(), + ) + verify_input = object() + + def record(phase, current): + self.assertTrue(current.dp_spec_prefill_coordination_applied) + counts = plan.phase_counts(phase)[0] + if phase != "target" and worker._draft_worker.draft_owns_attention: + counts = [counts[rank]] + self.assertEqual(current.global_num_tokens, counts) + self.assertFalse(current.can_run_decode_cuda_graph) + self.assertFalse(current.can_run_dp_prefill_cuda_graph) + events.append(phase) + + def draft(current): + record("draft", current) + self.assertIs(current, idle if is_prefill else batch) + return verify_input + + def target(current, **kwargs): + record("target", current) + if not is_prefill: + self.assertIs(current.spec_info, verify_input) + self.assertEqual(kwargs["grammar_barrier"], "grammar") + return result + + def extend(current, *args): + record("draft_extend", current) + return object() + + def draft_context(group, *, owns_attention): + self.assertEqual(owns_attention, bool(rank % 2)) + return contextlib.nullcontext() + + worker = object.__new__(EAGLEWorkerV2) + worker.device = "cpu" + worker.topk = 1 + worker.speculative_algorithm = SpeculativeAlgorithm.EAGLE + worker._target_worker = SimpleNamespace(forward_batch_generation=target) + worker._draft_worker = SimpleNamespace( + draft_runner=SimpleNamespace(tp_group=None), + draft_owns_attention=bool(rank % 2), + draft_tp_context=draft_context, + draft=draft, + _draft_extend_for_prefill=extend, + _draft_extend_for_decode=extend, + ) + worker.verify = target + with contextlib.ExitStack() as stack: + stack.enter_context( + patch( + f"{WORKER_MODULE}.get_parallel", + return_value=SimpleNamespace(attn_dp_rank=rank), + ) + ) + stack.enter_context( + patch( + f"{WORKER_MODULE}.ScheduleBatch.init_new", return_value=idle + ) + ) + stack.enter_context( + patch( + f"{WORKER_MODULE}.get_draft_recurrent_hidden_state_spec", + return_value=(16, torch.float32), + ) + ) + stack.enter_context( + patch( + f"{WORKER_MODULE}.EagleDraftInput.create_idle_input", + return_value=object(), + ) + ) + for name in ( + "speculative_moe_backend_context", + "speculative_moe_a2a_backend_context", + "spec_stage_span", + ): + stack.enter_context( + patch( + f"{WORKER_MODULE}.{name}", + side_effect=lambda *args: contextlib.nullcontext(), + ) + ) + actual = worker._forward_dp_spec_prefill_coordination( + batch, + plan, + lambda value: events.append("publish"), + "grammar", + None, + ) + self.assertIs(actual, result) + self.assertEqual(events, ["draft", "target", "publish", "draft_extend"]) + if is_prefill: + self.assertIn(idle, result.extra_keep_alive_refs) + idle.prepare_for_idle.assert_called_once() + + def test_local_mixed_batch_is_rejected_before_draft(self): + worker = object.__new__(EAGLEWorkerV2) + worker._draft_worker = SimpleNamespace(draft_owns_attention=False) + batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + decoding_reqs=[object()], + global_num_tokens=[1, 1], + ) + with patch( + f"{WORKER_MODULE}.get_parallel", + return_value=SimpleNamespace(attn_dp_rank=0), + ): + with self.assertRaisesRegex(RuntimeError, "Local mixed"): + worker._forward_dp_spec_prefill_coordination( + batch, make_plan(), None, None, None + ) + + def test_disabled_feature_retains_existing_dispatch_without_phase_metadata(self): + worker = object.__new__(EAGLEWorkerV2) + worker.enable_dp_spec_prefill_coordination = False + worker._forward_prefill_batch = MagicMock(return_value=object()) + worker._forward_dp_spec_prefill_coordination = MagicMock() + batch = SimpleNamespace( + is_extend_in_batch=True, + forward_mode=ForwardMode.DECODE, + ) + result = worker.forward_batch_generation(batch) + self.assertIs(result, worker._forward_prefill_batch.return_value) + worker._forward_dp_spec_prefill_coordination.assert_not_called() + + def test_uniform_prefill_retains_existing_dispatch(self): + worker = object.__new__(EAGLEWorkerV2) + worker.enable_dp_spec_prefill_coordination = True + worker.topk = 1 + worker.speculative_num_draft_tokens = 4 + worker._forward_prefill_batch = MagicMock(return_value=object()) + worker._forward_dp_spec_prefill_coordination = MagicMock() + batch = SimpleNamespace( + is_extend_in_batch=True, + forward_mode=ForwardMode.EXTEND, + dp_spec_prefill_coordination_metadata=( + [4096, 0], + [1, 0], + torch.tensor([1, 0]), + ), + ) + result = worker.forward_batch_generation(batch) + self.assertIs(result, worker._forward_prefill_batch.return_value) + worker._forward_dp_spec_prefill_coordination.assert_not_called() + + +if __name__ == "__main__": + unittest.main()