[DSpark] Merge the draft TP sync sites into dspark-draft (#43669)
This commit is contained in:
@@ -1412,7 +1412,7 @@ class Envs:
|
||||
# Which speculative decisions rank 0 broadcasts to its TP group; narrowing
|
||||
# it under live traffic isolates where ranks actually diverge. Comma
|
||||
# separated presets ("all", "rng", "init", "off"), or SpecTpSyncSite slugs
|
||||
# and numbers, each negatable with a leading "-": "all,-dspark-plan,-6".
|
||||
# and numbers, each negatable with a leading "-": "all,-dspark-plan,-dspark-draft".
|
||||
SGLANG_SPEC_TP_SYNC = EnvStr("all")
|
||||
# Kill-switch for the draft-extend cuda graph. Draft extend then always runs
|
||||
# eager. Escape hatch for setups where the capture's memory pool costs more
|
||||
|
||||
@@ -183,7 +183,6 @@ def draft_markov_block(
|
||||
step_temperatures: torch.Tensor,
|
||||
noise: DraftNoise,
|
||||
tp_sync: SpecTpSync,
|
||||
site: SpecTpSyncSite,
|
||||
logits_out: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Returns the drafted tokens and the [bs, gamma, vocab] step logits,
|
||||
@@ -210,7 +209,7 @@ def draft_markov_block(
|
||||
step=step,
|
||||
logits_out=logits_out[:, step],
|
||||
)
|
||||
prev_tokens = tp_sync.sync(site, prev_tokens)
|
||||
prev_tokens = tp_sync.sync(SpecTpSyncSite.DSPARK_DRAFT, prev_tokens)
|
||||
draft_tokens.append(prev_tokens)
|
||||
noise.offset.add_(1)
|
||||
return torch.stack(draft_tokens, dim=1), logits_out
|
||||
@@ -237,7 +236,6 @@ def sample_draft_block(
|
||||
) -> DraftBlockResult:
|
||||
bs = base_logits.shape[0]
|
||||
greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device)
|
||||
any_sampling = sampling_info is not None and not sampling_info.is_all_greedy
|
||||
temperatures = resolve_draft_temperatures(
|
||||
bs=bs, sampling_info=sampling_info, device=device
|
||||
)
|
||||
@@ -249,11 +247,6 @@ def sample_draft_block(
|
||||
step_temperatures=temperatures.masked_fill(greedy_mask, 0.0),
|
||||
noise=noise,
|
||||
tp_sync=tp_sync,
|
||||
site=(
|
||||
SpecTpSyncSite.DSPARK_DRAFT_SAMPLE
|
||||
if any_sampling
|
||||
else SpecTpSyncSite.DSPARK_DRAFT_GREEDY
|
||||
),
|
||||
)
|
||||
return DraftBlockResult(
|
||||
draft_tokens=draft_tokens,
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.srt.speculative.dspark_components.dspark_draft import (
|
||||
select_draft_hidden_without_anchor,
|
||||
step_logits_width,
|
||||
)
|
||||
from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite
|
||||
from sglang.srt.speculative.spec_tp_sync import SpecTpSync
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -112,7 +112,6 @@ class DsparkDraftSampler:
|
||||
step_temperatures=self.step_temperatures[:bs],
|
||||
noise=self._noise,
|
||||
tp_sync=self._tp_sync,
|
||||
site=SpecTpSyncSite.DSPARK_GRAPH_SAMPLE,
|
||||
logits_out=self.step_logits[:bs],
|
||||
)
|
||||
self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1))
|
||||
|
||||
@@ -19,16 +19,13 @@ class SpecTpSyncSite(IntEnum):
|
||||
|
||||
# -- DSpark --
|
||||
DSPARK_MEM = 1 # gates the draft graph capture
|
||||
DSPARK_DRAFT_GREEDY = 2
|
||||
DSPARK_DRAFT_SAMPLE = 3
|
||||
DSPARK_DRAFT_MULTINOMIAL = 4
|
||||
DSPARK_GRAPH_SAMPLE = 5 # in-graph philox, redrawn per replay
|
||||
DSPARK_GRAPH_GREEDY = 6
|
||||
DSPARK_PLAN = 7
|
||||
DSPARK_ACCEPT_GREEDY = 8
|
||||
DSPARK_ACCEPT_SAMPLE = 9
|
||||
DSPARK_ACCEPT_GRAPH = 10
|
||||
DSPARK_TARGET = 11
|
||||
# A fresh number, so the retired 2-6 cannot silently select it.
|
||||
DSPARK_DRAFT = 17
|
||||
|
||||
# -- DFlash --
|
||||
DFLASH_MEM = 12
|
||||
@@ -47,9 +44,6 @@ _INIT = frozenset({SpecTpSyncSite.DSPARK_MEM, SpecTpSyncSite.DFLASH_MEM})
|
||||
# Sites that draw from the RNG, so they can differ under identical logits.
|
||||
_RNG = frozenset(
|
||||
{
|
||||
SpecTpSyncSite.DSPARK_DRAFT_SAMPLE,
|
||||
SpecTpSyncSite.DSPARK_DRAFT_MULTINOMIAL,
|
||||
SpecTpSyncSite.DSPARK_GRAPH_SAMPLE,
|
||||
SpecTpSyncSite.DSPARK_ACCEPT_SAMPLE,
|
||||
SpecTpSyncSite.DSPARK_TARGET,
|
||||
SpecTpSyncSite.DFLASH_SELECTOR,
|
||||
@@ -69,11 +63,36 @@ _PRESETS = {
|
||||
|
||||
_BY_SLUG = {site.slug: site for site in SpecTpSyncSite}
|
||||
_BY_NUMBER = {str(int(site)): site for site in SpecTpSyncSite}
|
||||
# Retired handles of the per-path DSpark draft sites, merged into DSPARK_DRAFT.
|
||||
_RETIRED = {
|
||||
token: SpecTpSyncSite.DSPARK_DRAFT
|
||||
for token in (
|
||||
"dspark-draft-greedy",
|
||||
"dspark-draft-sample",
|
||||
"dspark-draft-multinomial",
|
||||
"dspark-graph-sample",
|
||||
"dspark-graph-greedy",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
"6",
|
||||
)
|
||||
}
|
||||
assert not _RETIRED.keys() & (_BY_SLUG.keys() | _BY_NUMBER.keys())
|
||||
|
||||
|
||||
def _resolve(name: str) -> frozenset[SpecTpSyncSite]:
|
||||
if name in _PRESETS:
|
||||
return _PRESETS[name]
|
||||
if name in _RETIRED:
|
||||
site = _RETIRED[name]
|
||||
logger.warning(
|
||||
"SGLANG_SPEC_TP_SYNC token %r is retired; it selects %r.",
|
||||
name,
|
||||
site.slug,
|
||||
)
|
||||
return frozenset({site})
|
||||
site = _BY_SLUG.get(name) or _BY_NUMBER.get(name)
|
||||
if site is None:
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user