From 3cc23b3b86c2f1583a316a8466c853e89aced06f Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sun, 11 Oct 2026 15:17:47 -0700 Subject: [PATCH] [DSpark] Merge the draft TP sync sites into `dspark-draft` (#43669) --- python/sglang/srt/environ.py | 2 +- .../dspark_components/dspark_draft.py | 9 +---- .../dspark_components/dspark_draft_sampler.py | 3 +- python/sglang/srt/speculative/spec_tp_sync.py | 35 ++++++++++++++----- .../spec/dspark/test_dspark_draft_graph.py | 2 -- .../registered/unit/spec/test_spec_tp_sync.py | 30 ++++++++++++++++ 6 files changed, 60 insertions(+), 21 deletions(-) create mode 100644 test/registered/unit/spec/test_spec_tp_sync.py diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index a54618045f..a2dd2f1248 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1412,7 +1412,7 @@ class Envs: # Which speculative decisions rank 0 broadcasts to its TP group; narrowing # it under live traffic isolates where ranks actually diverge. Comma # separated presets ("all", "rng", "init", "off"), or SpecTpSyncSite slugs - # and numbers, each negatable with a leading "-": "all,-dspark-plan,-6". + # and numbers, each negatable with a leading "-": "all,-dspark-plan,-dspark-draft". SGLANG_SPEC_TP_SYNC = EnvStr("all") # Kill-switch for the draft-extend cuda graph. Draft extend then always runs # eager. Escape hatch for setups where the capture's memory pool costs more diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft.py b/python/sglang/srt/speculative/dspark_components/dspark_draft.py index 714406bb0e..9132776636 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft.py @@ -183,7 +183,6 @@ def draft_markov_block( step_temperatures: torch.Tensor, noise: DraftNoise, tp_sync: SpecTpSync, - site: SpecTpSyncSite, logits_out: Optional[torch.Tensor] = None, ) -> tuple[torch.Tensor, torch.Tensor]: """Returns the drafted tokens and the [bs, gamma, vocab] step logits, @@ -210,7 +209,7 @@ def draft_markov_block( step=step, logits_out=logits_out[:, step], ) - prev_tokens = tp_sync.sync(site, prev_tokens) + prev_tokens = tp_sync.sync(SpecTpSyncSite.DSPARK_DRAFT, prev_tokens) draft_tokens.append(prev_tokens) noise.offset.add_(1) return torch.stack(draft_tokens, dim=1), logits_out @@ -237,7 +236,6 @@ def sample_draft_block( ) -> DraftBlockResult: bs = base_logits.shape[0] greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device) - any_sampling = sampling_info is not None and not sampling_info.is_all_greedy temperatures = resolve_draft_temperatures( bs=bs, sampling_info=sampling_info, device=device ) @@ -249,11 +247,6 @@ def sample_draft_block( step_temperatures=temperatures.masked_fill(greedy_mask, 0.0), noise=noise, tp_sync=tp_sync, - site=( - SpecTpSyncSite.DSPARK_DRAFT_SAMPLE - if any_sampling - else SpecTpSyncSite.DSPARK_DRAFT_GREEDY - ), ) return DraftBlockResult( draft_tokens=draft_tokens, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py index 1678bd9f4c..4332bae31b 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py @@ -11,7 +11,7 @@ from sglang.srt.speculative.dspark_components.dspark_draft import ( select_draft_hidden_without_anchor, step_logits_width, ) -from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite +from sglang.srt.speculative.spec_tp_sync import SpecTpSync logger = logging.getLogger(__name__) @@ -112,7 +112,6 @@ class DsparkDraftSampler: step_temperatures=self.step_temperatures[:bs], noise=self._noise, tp_sync=self._tp_sync, - site=SpecTpSyncSite.DSPARK_GRAPH_SAMPLE, logits_out=self.step_logits[:bs], ) self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1)) diff --git a/python/sglang/srt/speculative/spec_tp_sync.py b/python/sglang/srt/speculative/spec_tp_sync.py index 40bc551bfd..0c6b2c2da3 100644 --- a/python/sglang/srt/speculative/spec_tp_sync.py +++ b/python/sglang/srt/speculative/spec_tp_sync.py @@ -19,16 +19,13 @@ class SpecTpSyncSite(IntEnum): # -- DSpark -- DSPARK_MEM = 1 # gates the draft graph capture - DSPARK_DRAFT_GREEDY = 2 - DSPARK_DRAFT_SAMPLE = 3 - DSPARK_DRAFT_MULTINOMIAL = 4 - DSPARK_GRAPH_SAMPLE = 5 # in-graph philox, redrawn per replay - DSPARK_GRAPH_GREEDY = 6 DSPARK_PLAN = 7 DSPARK_ACCEPT_GREEDY = 8 DSPARK_ACCEPT_SAMPLE = 9 DSPARK_ACCEPT_GRAPH = 10 DSPARK_TARGET = 11 + # A fresh number, so the retired 2-6 cannot silently select it. + DSPARK_DRAFT = 17 # -- DFlash -- DFLASH_MEM = 12 @@ -47,9 +44,6 @@ _INIT = frozenset({SpecTpSyncSite.DSPARK_MEM, SpecTpSyncSite.DFLASH_MEM}) # Sites that draw from the RNG, so they can differ under identical logits. _RNG = frozenset( { - SpecTpSyncSite.DSPARK_DRAFT_SAMPLE, - SpecTpSyncSite.DSPARK_DRAFT_MULTINOMIAL, - SpecTpSyncSite.DSPARK_GRAPH_SAMPLE, SpecTpSyncSite.DSPARK_ACCEPT_SAMPLE, SpecTpSyncSite.DSPARK_TARGET, SpecTpSyncSite.DFLASH_SELECTOR, @@ -69,11 +63,36 @@ _PRESETS = { _BY_SLUG = {site.slug: site for site in SpecTpSyncSite} _BY_NUMBER = {str(int(site)): site for site in SpecTpSyncSite} +# Retired handles of the per-path DSpark draft sites, merged into DSPARK_DRAFT. +_RETIRED = { + token: SpecTpSyncSite.DSPARK_DRAFT + for token in ( + "dspark-draft-greedy", + "dspark-draft-sample", + "dspark-draft-multinomial", + "dspark-graph-sample", + "dspark-graph-greedy", + "2", + "3", + "4", + "5", + "6", + ) +} +assert not _RETIRED.keys() & (_BY_SLUG.keys() | _BY_NUMBER.keys()) def _resolve(name: str) -> frozenset[SpecTpSyncSite]: if name in _PRESETS: return _PRESETS[name] + if name in _RETIRED: + site = _RETIRED[name] + logger.warning( + "SGLANG_SPEC_TP_SYNC token %r is retired; it selects %r.", + name, + site.slug, + ) + return frozenset({site}) site = _BY_SLUG.get(name) or _BY_NUMBER.get(name) if site is None: raise ValueError( diff --git a/test/registered/spec/dspark/test_dspark_draft_graph.py b/test/registered/spec/dspark/test_dspark_draft_graph.py index 746a325487..d46146ecac 100644 --- a/test/registered/spec/dspark/test_dspark_draft_graph.py +++ b/test/registered/spec/dspark/test_dspark_draft_graph.py @@ -12,7 +12,6 @@ from sglang.srt.speculative.dspark_components.dspark_draft import ( from sglang.srt.speculative.dspark_components.dspark_draft_sampler import ( DsparkDraftSampler, ) -from sglang.srt.speculative.spec_tp_sync import SpecTpSyncSite from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -47,7 +46,6 @@ class TestDraftMarkovBlockGraph(CustomTestCase): step_temperatures=temperatures, noise=noise, tp_sync=SimpleNamespace(sync=lambda site, value: value), - site=SpecTpSyncSite.DSPARK_GRAPH_SAMPLE, logits_out=logits_out, ) tokens_out.copy_(tokens) diff --git a/test/registered/unit/spec/test_spec_tp_sync.py b/test/registered/unit/spec/test_spec_tp_sync.py new file mode 100644 index 0000000000..c5d96e1805 --- /dev/null +++ b/test/registered/unit/spec/test_spec_tp_sync.py @@ -0,0 +1,30 @@ +import unittest + +from sglang.srt.speculative.spec_tp_sync import SpecTpSyncSite, parse_spec_tp_sync +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +DRAFT = SpecTpSyncSite.DSPARK_DRAFT + + +class TestParseSpecTpSync(CustomTestCase): + def test_retired_draft_handles_select_the_merged_site(self): + for token in ("dspark-graph-greedy", "dspark-draft-sample", "2", "6"): + with self.subTest(token=token): + self.assertEqual(parse_spec_tp_sync(token), frozenset({DRAFT})) + + def test_negating_a_retired_handle_drops_the_merged_site(self): + sites = parse_spec_tp_sync("all,-6") + self.assertNotIn(DRAFT, sites) + self.assertEqual(sites, frozenset(SpecTpSyncSite) - {DRAFT}) + + def test_draft_noise_is_not_an_rng_site(self): + # Ranks with the same logits draw the same draft noise. + self.assertNotIn(DRAFT, parse_spec_tp_sync("rng")) + self.assertIn(DRAFT, parse_spec_tp_sync("dspark-draft")) + + +if __name__ == "__main__": + unittest.main()