[DSpark] Merge the draft TP sync sites into dspark-draft (#43669)

This commit is contained in:
Liangsheng Yin
2026-10-11 15:17:47 -07:00
committed by GitHub
parent 00ba90fd1a
commit 3cc23b3b86
6 changed files with 60 additions and 21 deletions
+1 -1
View File
@@ -1412,7 +1412,7 @@ class Envs:
# Which speculative decisions rank 0 broadcasts to its TP group; narrowing
# it under live traffic isolates where ranks actually diverge. Comma
# separated presets ("all", "rng", "init", "off"), or SpecTpSyncSite slugs
# and numbers, each negatable with a leading "-": "all,-dspark-plan,-6".
# and numbers, each negatable with a leading "-": "all,-dspark-plan,-dspark-draft".
SGLANG_SPEC_TP_SYNC = EnvStr("all")
# Kill-switch for the draft-extend cuda graph. Draft extend then always runs
# eager. Escape hatch for setups where the capture's memory pool costs more
@@ -183,7 +183,6 @@ def draft_markov_block(
step_temperatures: torch.Tensor,
noise: DraftNoise,
tp_sync: SpecTpSync,
site: SpecTpSyncSite,
logits_out: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Returns the drafted tokens and the [bs, gamma, vocab] step logits,
@@ -210,7 +209,7 @@ def draft_markov_block(
step=step,
logits_out=logits_out[:, step],
)
prev_tokens = tp_sync.sync(site, prev_tokens)
prev_tokens = tp_sync.sync(SpecTpSyncSite.DSPARK_DRAFT, prev_tokens)
draft_tokens.append(prev_tokens)
noise.offset.add_(1)
return torch.stack(draft_tokens, dim=1), logits_out
@@ -237,7 +236,6 @@ def sample_draft_block(
) -> DraftBlockResult:
bs = base_logits.shape[0]
greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device)
any_sampling = sampling_info is not None and not sampling_info.is_all_greedy
temperatures = resolve_draft_temperatures(
bs=bs, sampling_info=sampling_info, device=device
)
@@ -249,11 +247,6 @@ def sample_draft_block(
step_temperatures=temperatures.masked_fill(greedy_mask, 0.0),
noise=noise,
tp_sync=tp_sync,
site=(
SpecTpSyncSite.DSPARK_DRAFT_SAMPLE
if any_sampling
else SpecTpSyncSite.DSPARK_DRAFT_GREEDY
),
)
return DraftBlockResult(
draft_tokens=draft_tokens,
@@ -11,7 +11,7 @@ from sglang.srt.speculative.dspark_components.dspark_draft import (
select_draft_hidden_without_anchor,
step_logits_width,
)
from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite
from sglang.srt.speculative.spec_tp_sync import SpecTpSync
logger = logging.getLogger(__name__)
@@ -112,7 +112,6 @@ class DsparkDraftSampler:
step_temperatures=self.step_temperatures[:bs],
noise=self._noise,
tp_sync=self._tp_sync,
site=SpecTpSyncSite.DSPARK_GRAPH_SAMPLE,
logits_out=self.step_logits[:bs],
)
self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1))
+27 -8
View File
@@ -19,16 +19,13 @@ class SpecTpSyncSite(IntEnum):
# -- DSpark --
DSPARK_MEM = 1 # gates the draft graph capture
DSPARK_DRAFT_GREEDY = 2
DSPARK_DRAFT_SAMPLE = 3
DSPARK_DRAFT_MULTINOMIAL = 4
DSPARK_GRAPH_SAMPLE = 5 # in-graph philox, redrawn per replay
DSPARK_GRAPH_GREEDY = 6
DSPARK_PLAN = 7
DSPARK_ACCEPT_GREEDY = 8
DSPARK_ACCEPT_SAMPLE = 9
DSPARK_ACCEPT_GRAPH = 10
DSPARK_TARGET = 11
# A fresh number, so the retired 2-6 cannot silently select it.
DSPARK_DRAFT = 17
# -- DFlash --
DFLASH_MEM = 12
@@ -47,9 +44,6 @@ _INIT = frozenset({SpecTpSyncSite.DSPARK_MEM, SpecTpSyncSite.DFLASH_MEM})
# Sites that draw from the RNG, so they can differ under identical logits.
_RNG = frozenset(
{
SpecTpSyncSite.DSPARK_DRAFT_SAMPLE,
SpecTpSyncSite.DSPARK_DRAFT_MULTINOMIAL,
SpecTpSyncSite.DSPARK_GRAPH_SAMPLE,
SpecTpSyncSite.DSPARK_ACCEPT_SAMPLE,
SpecTpSyncSite.DSPARK_TARGET,
SpecTpSyncSite.DFLASH_SELECTOR,
@@ -69,11 +63,36 @@ _PRESETS = {
_BY_SLUG = {site.slug: site for site in SpecTpSyncSite}
_BY_NUMBER = {str(int(site)): site for site in SpecTpSyncSite}
# Retired handles of the per-path DSpark draft sites, merged into DSPARK_DRAFT.
_RETIRED = {
token: SpecTpSyncSite.DSPARK_DRAFT
for token in (
"dspark-draft-greedy",
"dspark-draft-sample",
"dspark-draft-multinomial",
"dspark-graph-sample",
"dspark-graph-greedy",
"2",
"3",
"4",
"5",
"6",
)
}
assert not _RETIRED.keys() & (_BY_SLUG.keys() | _BY_NUMBER.keys())
def _resolve(name: str) -> frozenset[SpecTpSyncSite]:
if name in _PRESETS:
return _PRESETS[name]
if name in _RETIRED:
site = _RETIRED[name]
logger.warning(
"SGLANG_SPEC_TP_SYNC token %r is retired; it selects %r.",
name,
site.slug,
)
return frozenset({site})
site = _BY_SLUG.get(name) or _BY_NUMBER.get(name)
if site is None:
raise ValueError(