diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py index 539188ba15..d2f497396d 100644 --- a/python/sglang/srt/arg_groups/speculative_hook.py +++ b/python/sglang/srt/arg_groups/speculative_hook.py @@ -221,11 +221,11 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None: if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get(): if ( - cfg.speculative_algorithm not in ("EAGLE", "EAGLE3") + cfg.speculative_algorithm not in ("EAGLE", "EAGLE3", "DSPARK") or cfg.enable_multi_layer_eagle ): raise ValueError( - "DP spec/prefill coordination requires single-layer EAGLE or EAGLE3" + "DP spec/prefill coordination requires single-layer EAGLE, EAGLE3 or DSPARK" ) if cfg.speculative_adaptive: @@ -598,7 +598,10 @@ def _handle_dspark(server_args: ServerArgs) -> None: "(built-in TP MoE), 'megamoe', or 'mori', got " f"{cfg.moe_a2a_backend!r}." ) - if not _is_npu and cfg.moe_a2a_backend != "none": + if not _is_npu and ( + cfg.moe_a2a_backend != "none" + or envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get() + ): from sglang.srt.speculative.ragged_verify import ( RaggedVerifyMode, read_ragged_verify_mode, @@ -606,8 +609,7 @@ def _handle_dspark(server_args: ServerArgs) -> None: if read_ragged_verify_mode() is not RaggedVerifyMode.STATIC: raise ValueError( - "DSpark with dp attention + " - f"moe_a2a_backend={cfg.moe_a2a_backend!r} requires " + "DSpark DP MoE or prefill coordination requires " "SGLANG_RAGGED_VERIFY_MODE=static." ) if cfg.attn_cp_size > 1: diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft.py b/python/sglang/srt/speculative/dspark_components/dspark_draft.py index ccce0b0bc7..872eb33117 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft.py @@ -478,6 +478,10 @@ class DraftBlockProposer: # The dense DSpark draft still reuses the target batch's graph tier. # Set graph eligibility before the DP-MoE-only metadata early return. forward_batch.can_run_decode_cuda_graph = batch.can_run_decode_cuda_graph + forward_batch.is_extend_in_batch = batch.is_extend_in_batch + forward_batch.dp_spec_prefill_coordination_applied = ( + batch.dp_spec_prefill_coordination_applied + ) device = self.draft_model_runner.device num_tokens = forward_batch.input_ids.numel() if self._num_token_non_padded is not None: @@ -490,11 +494,15 @@ class DraftBlockProposer: # them separate from global_num_tokens_cpu below, which is scaled into # draft-token units for DP/MoE synchronization. forward_batch.original_global_num_tokens_cpu = batch.global_num_tokens - gnt, gnt_logprob = spec_scale_global_num_tokens( - self._draft_block_spec_info, - batch.global_num_tokens, - batch.global_num_tokens_for_logprob, - ) + if batch.dp_spec_prefill_coordination_applied: + gnt = batch.global_num_tokens + gnt_logprob = batch.global_num_tokens_for_logprob + else: + gnt, gnt_logprob = spec_scale_global_num_tokens( + self._draft_block_spec_info, + batch.global_num_tokens, + batch.global_num_tokens_for_logprob, + ) forward_batch.original_global_num_tokens_cpu = batch.global_num_tokens num_tokens = forward_batch.input_ids.numel() num_token_non_padded = _make_num_token_non_padded(num_tokens, device) diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index a63e3198b7..e0a5596f5a 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -32,6 +32,9 @@ from sglang.srt.runtime_context import ( from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.base_spec_worker import BaseSpecWorker from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2 +from sglang.srt.speculative.dp_spec_prefill_coordination import ( + DPSpecPrefillCoordinationPlan, +) from sglang.srt.speculative.draft_worker_common import ( build_block_pos_offsets, build_draft_tp_worker, @@ -151,6 +154,9 @@ class DSparkWorkerV2(BaseSpecWorker): self.page_size = get_schedule().page_size self.device = target_worker.device self._draft_worker = None + self.enable_dp_spec_prefill_coordination = ( + envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get() + ) self._hosts_draft = get_parallel().pp_group.is_last_rank if not self._hosts_draft: return @@ -571,12 +577,46 @@ class DSparkWorkerV2(BaseSpecWorker): on_publish(batch_output.new_seq_lens) return batch_output if batch.forward_mode.is_extend() or batch.is_extend_in_batch: + if batch.is_extend_in_batch and self.enable_dp_spec_prefill_coordination: + plan = DPSpecPrefillCoordinationPlan( + *batch.dp_spec_prefill_coordination_metadata, + draft_width=self._proposer.query_token_num, + verify_width=self.verify_num_draft_tokens, + ) + if plan.heterogeneous: + return self._forward_dp_spec_prefill_coordination( + batch, plan, on_publish, grammar_barrier, pp_proxy_tensors + ) self._verify_planner.note_non_decode_step() self._observers.note_prefill_step() return self._forward_prefill(batch, on_publish, pp_proxy_tensors) return self._forward_decode(batch, on_publish, grammar_barrier) + def _forward_dp_spec_prefill_coordination( + self, batch, plan, on_publish, grammar_barrier, pp_proxy_tensors + ): + """Pair local prefill with peer verification after the shared draft pass.""" + if not batch.forward_mode.is_extend(): + return self._forward_decode( + batch, on_publish, grammar_barrier, coordination_plan=plan + ) + + rank = get_parallel().attn_dp_rank + target_local_only = len(batch.global_num_tokens) == 1 + plan.apply( + batch, + "draft", + rank, + local_only=target_local_only or self._draft_dp_context_enabled, + ) + with self._draft_context(): + self._proposer.run_idle_participation(batch) + plan.apply(batch, "target", rank, local_only=target_local_only) + self._verify_planner.note_non_decode_step() + self._observers.note_prefill_step() + return self._forward_prefill(batch, on_publish, pp_proxy_tensors) + def _forward_prefill( self, batch: ScheduleBatch, @@ -732,7 +772,11 @@ class DSparkWorkerV2(BaseSpecWorker): ) def _forward_decode( - self, batch: ScheduleBatch, on_publish, grammar_barrier=None + self, + batch: ScheduleBatch, + on_publish, + grammar_barrier=None, + coordination_plan=None, ) -> GenerationBatchResult: if batch.spec_info is None: batch.spec_info = DFlashDraftInputV2.create_idle_input(device=self.device) @@ -742,11 +786,32 @@ class DSparkWorkerV2(BaseSpecWorker): "DSpark spec-v2 expected DFlashDraftInputV2 state on the running batch." ) + global_num_reqs = ( + max(batch.global_num_tokens) + if self._draft_is_moe + and get_parallel().enable_dp_attention + and batch.global_num_tokens is not None + else None + ) + if coordination_plan is not None: + rank = get_parallel().attn_dp_rank + target_local_only = len(batch.global_num_tokens) == 1 + coordination_plan.apply( + batch, + "draft", + rank, + local_only=target_local_only or self._draft_dp_context_enabled, + ) + if batch.forward_mode.is_idle(): self._observers.note_idle_decode_step() if get_parallel().enable_dp_attention: if self._draft_is_moe: self._proposer.run_idle_participation(batch) + if coordination_plan is not None: + coordination_plan.apply( + batch, "target", rank, local_only=target_local_only + ) self._verify_executor.run_idle_participation( batch=batch, idle_layout=self._idle_verify_ragged_layout(batch) ) @@ -785,6 +850,8 @@ class DSparkWorkerV2(BaseSpecWorker): draft_block_ids = proposal.draft_block_ids draft_block = proposal.draft_block draft_tokens = draft_block.draft_tokens + if coordination_plan is not None: + coordination_plan.apply(batch, "target", rank, local_only=target_local_only) confidence = proposal.confidence if confidence is None: @@ -802,13 +869,6 @@ class DSparkWorkerV2(BaseSpecWorker): req_pool_indices=batch.req_pool_indices, ) - global_num_reqs = ( - max(batch.global_num_tokens) - if self._draft_is_moe - and get_parallel().enable_dp_attention - and batch.global_num_tokens is not None - else None - ) layout = self._verify_planner.schedule_layout( req_pool_indices=batch.req_pool_indices, prefix_lens=prefix_lens, diff --git a/test/registered/spec/dspark/test_dspark_dp_spec_prefill_coordination.py b/test/registered/spec/dspark/test_dspark_dp_spec_prefill_coordination.py new file mode 100644 index 0000000000..8e7fbcb47f --- /dev/null +++ b/test/registered/spec/dspark/test_dspark_dp_spec_prefill_coordination.py @@ -0,0 +1,160 @@ +import json +import os +import threading +import time +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor + +import requests + +from sglang.srt.utils import is_sm100_supported +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + kill_process_tree, + popen_launch_server, +) + +register_cuda_ci(est_time=240, stage="base-c", runner_config="4-gpu-h100") + + +class TestDSparkDPSpecPrefillCoordination(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.base_url = os.environ.get("SGLANG_TEST_DSPARK_COORDINATION_URL") + cls.process = None + cls.model = os.environ.get("SGLANG_TEST_DSPARK_MODEL", "Qwen/Qwen3-14B") + if cls.base_url is None: + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "DSPARK", + "--speculative-draft-model-path", + "deepseek-ai/dspark_qwen3_14b_block7", + "--tp-size", + "2", + "--dp-size", + "2", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--attention-backend", + "trtllm_mha" if is_sm100_supported() else "fa3", + "--speculative-draft-attention-backend", + "fa4" if is_sm100_supported() else "fa3", + "--page-size", + "1", + "--mem-fraction-static", + "0.75", + "--cuda-graph-max-bs-decode", + "4", + "--cuda-graph-backend-prefill", + "disabled", + "--enable-mixed-chunk", + "--chunked-prefill-size", + "16384", + ], + env={ + "SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION": "1", + "SGLANG_RAGGED_VERIFY_MODE": "static", + "SGLANG_SIMULATE_ACC_LEN": "-1", + "SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY": "1", + }, + ) + response = requests.get(cls.base_url + "/server_info", timeout=30) + response.raise_for_status() + cls.incremental_streaming = response.json()["incremental_streaming_output"] + cls.prompt = "Count from 1 to 1000, separated by commas.\n1, 2, 3," + + @classmethod + def tearDownClass(cls): + if cls.process is not None: + kill_process_tree(cls.process.pid) + + def generate(self, prompt, rank, tokens, started=None): + payload = { + "text": prompt, + "routed_dp_rank": rank, + "stream": started is not None, + "sampling_params": { + "temperature": 0, + "max_new_tokens": tokens, + "ignore_eos": True, + }, + } + with requests.post( + self.base_url + "/generate", + json=payload, + stream=started is not None, + timeout=300, + ) as response: + response.raise_for_status() + if started is None: + result = response.json() + else: + result = None + chunks = [] + for line in response.iter_lines(chunk_size=1): + if not line.startswith(b"data:"): + continue + data = line[5:].strip() + if data == b"[DONE]": + break + result = json.loads(data) + chunks.append(result["text"]) + if result.get("meta_info", {}).get("completion_tokens", 0) >= 64: + started.set() + self.assertIsNotNone(result) + if self.incremental_streaming: + result["text"] = "".join(chunks) + self.assertEqual(result["meta_info"]["completion_tokens"], tokens) + self.assertTrue(result["text"]) + return result["text"], time.monotonic() + + def test_mixed_prefill_and_speculative_peer_match_isolated_outputs(self): + expected = { + rank: self.generate(self.prompt, rank, 1024)[0] for rank in range(2) + } + for prefill_rank in range(2): + long_prompt = ( + f"Fresh context {uuid.uuid4().hex}:\n" + + "\n".join( + f"Entry {i}: apples are fruit; water is liquid; the sky appears blue." + for i in range(1536) + ) + + "\nEnd of context.\n" + + self.prompt + ) + started = [threading.Event(), threading.Event()] + with self.subTest(prefill_rank=prefill_rank), ThreadPoolExecutor(3) as pool: + decodes = [ + pool.submit(self.generate, self.prompt, rank, 1024, started[rank]) + for rank in range(2) + ] + for event in started: + self.assertTrue(event.wait(90), "Decode did not start") + self.assertTrue(all(not f.done() for f in decodes)) + prefill_started = time.monotonic() + prefill = pool.submit(self.generate, long_prompt, prefill_rank, 32) + results = [f.result(timeout=300) for f in decodes] + prefill_output, _ = prefill.result(timeout=300) + for rank, (output, completed) in enumerate(results): + self.assertGreater(completed, prefill_started) + self.assertEqual(output, expected[rank]) + self.assertEqual( + prefill_output, self.generate(long_prompt, prefill_rank, 32)[0] + ) + for rank in range(2): + self.assertEqual( + self.generate(self.prompt, rank, 1024)[0], expected[rank] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/spec/dspark/test_dspark_dp_tier.py b/test/registered/spec/dspark/test_dspark_dp_tier.py index 95f3a1ca88..2e804f3212 100644 --- a/test/registered/spec/dspark/test_dspark_dp_tier.py +++ b/test/registered/spec/dspark/test_dspark_dp_tier.py @@ -69,6 +69,8 @@ class TestDraftDpSyncMetadata(CustomTestCase): global_num_tokens=[1, 3, 0, 2], global_num_tokens_for_logprob=[1, 3, 0, 2], can_run_decode_cuda_graph=True, + is_extend_in_batch=False, + dp_spec_prefill_coordination_applied=False, ) with patch( diff --git a/test/registered/unit/spec/test_dspark_dp_spec_prefill_coordination.py b/test/registered/unit/spec/test_dspark_dp_spec_prefill_coordination.py new file mode 100644 index 0000000000..80e90e635f --- /dev/null +++ b/test/registered/unit/spec/test_dspark_dp_spec_prefill_coordination.py @@ -0,0 +1,141 @@ +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import torch + +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.speculative.dp_spec_prefill_coordination import ( + DPSpecPrefillCoordinationPlan, +) +from sglang.srt.speculative.dspark_components.dspark_draft import DraftBlockProposer +from sglang.srt.speculative.dspark_components.dspark_worker_v2 import DSparkWorkerV2 +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=8, suite="base-a-test-cpu") + +MODULE = "sglang.srt.speculative.dspark_components.dspark_worker_v2" + + +class TestDSparkDPSpecPrefillCoordination(CustomTestCase): + def test_draft_counts_are_not_scaled_twice(self): + for width in (6, 7): + for rank, local_tokens in ((0, 0), (1, 3 * width), (2, 0)): + with self.subTest(width=width, rank=rank): + plan = DPSpecPrefillCoordinationPlan( + [8192, 3, 0], [1, 3, 0], torch.tensor([1, 0, 0]), width, 7 + ) + batch = SimpleNamespace() + plan.apply(batch, "draft", rank, local_only=False) + proposer = object.__new__(DraftBlockProposer) + proposer._dp_moe_sync = True + proposer._num_token_non_padded = None + proposer.draft_model_runner = SimpleNamespace(device="cpu") + proposer._draft_block_spec_info = SimpleNamespace( + num_tokens_per_req=width, num_tokens_for_logprob_per_req=1 + ) + forward = SimpleNamespace(input_ids=torch.arange(local_tokens)) + with patch( + "sglang.srt.speculative.dspark_components.dspark_draft.enable_num_token_non_padded", + return_value=True, + ): + proposer._fill_dp_moe_sync_metadata(forward, batch) + self.assertEqual(forward.global_num_tokens_cpu, [0, 3 * width, 0]) + self.assertEqual( + forward.global_num_tokens_for_logprob_cpu, [0, 3 * width, 0] + ) + self.assertTrue(forward.dp_spec_prefill_coordination_applied) + self.assertTrue(forward.is_extend_in_batch) + self.assertFalse(forward.can_run_decode_cuda_graph) + plan.apply(batch, "target", rank, local_only=False) + self.assertEqual(batch.global_num_tokens, [8192, 21, 0]) + self.assertEqual(batch.global_num_tokens_for_logprob, [1, 21, 0]) + + def test_prefill_joins_draft_before_target_with_local_counts_preserved(self): + plan = DPSpecPrefillCoordinationPlan( + [8192, 3, 0], [1, 3, 0], torch.tensor([1, 0, 0]), 6, 7 + ) + for local_draft in (False, True): + with self.subTest(local_draft=local_draft): + batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + global_num_tokens=plan.counts, + ) + events = [] + worker = object.__new__(DSparkWorkerV2) + worker._draft_dp_context_enabled = local_draft + worker._draft_context = nullcontext + worker._verify_planner = MagicMock() + worker._observers = MagicMock() + + def draft(current): + self.assertIs(current, batch) + self.assertEqual( + current.global_num_tokens, [0] if local_draft else [0, 18, 0] + ) + self.assertFalse(current.can_run_decode_cuda_graph) + events.append("draft") + + def target(current, publish, proxy): + self.assertEqual(current.global_num_tokens, [8192, 21, 0]) + self.assertEqual(current.forward_mode, ForwardMode.EXTEND) + self.assertFalse(current.can_run_decode_cuda_graph) + events.append("target") + return "result" + + worker._proposer = SimpleNamespace(run_idle_participation=draft) + worker._forward_prefill = target + with patch( + f"{MODULE}.get_parallel", + return_value=SimpleNamespace(attn_dp_rank=0), + ): + result = worker._forward_dp_spec_prefill_coordination( + batch, plan, None, None, None + ) + self.assertEqual(result, "result") + self.assertEqual(events, ["draft", "target"]) + + def test_dispatch_keeps_uniform_and_disabled_paths(self): + cases = [ + (False, ForwardMode.EXTEND, True, [8192, 3], [1, 0], "prefill"), + (True, ForwardMode.EXTEND, True, [8192, 0], [1, 0], "prefill"), + (True, ForwardMode.DECODE, False, [3, 2], [0, 0], "decode"), + (True, ForwardMode.DECODE, True, [8192, 3], [1, 0], "coordinate"), + (True, ForwardMode.IDLE, True, [8192, 3, 0], [1, 0, 0], "coordinate"), + ] + for enabled, mode, extend, counts, prefills, expected in cases: + with self.subTest(enabled=enabled, mode=mode, expected=expected): + worker = object.__new__(DSparkWorkerV2) + worker._hosts_draft = True + worker.enable_dp_spec_prefill_coordination = enabled + worker.verify_num_draft_tokens = 7 + worker._proposer = SimpleNamespace(query_token_num=6) + worker._verify_planner = MagicMock() + worker._observers = MagicMock() + worker._forward_prefill = MagicMock(return_value="prefill") + worker._forward_decode = MagicMock(return_value="decode") + worker._forward_dp_spec_prefill_coordination = MagicMock( + return_value="coordinate" + ) + batch = SimpleNamespace( + forward_mode=mode, + is_extend_in_batch=extend, + dp_spec_prefill_coordination_metadata=( + counts, + counts, + torch.tensor(prefills), + ), + ) + self.assertEqual(worker.forward_batch_generation(batch), expected) + if expected == "coordinate": + plan = worker._forward_dp_spec_prefill_coordination.call_args.args[ + 1 + ] + self.assertEqual(plan.draft_width, 6) + self.assertEqual(plan.verify_width, 7) + + +if __name__ == "__main__": + unittest.main()