[DSpark] Add SGLANG_DSPARK_USE_GREEDY_DRAFT, an argmax draft accepted as one-hot (#43670)

This commit is contained in:
Liangsheng Yin
2026-10-11 15:18:59 -07:00
committed by GitHub
parent 3cc23b3b86
commit a65b776963
12 changed files with 261 additions and 30 deletions
@@ -19,11 +19,11 @@ class MarkovGreedyStep:
Computes ``argmax_v(base_logits[:, v] + dot(w2_weight[v, :], prev_embeds))``
in a single pass over the (vocab x rank) weight: no full-vocab bias or
step-logits materialization, no separate GEMV / add / two-pass argmax
launches. Numerics: the dot and the add accumulate in fp32, while the eager
path rounds the GEMV output and the add to bf16 before argmax — near-tie
winners can differ on rare steps. Drafts are proposals only (target verify
guards output correctness), so the impact is bounded to accept-rate noise
on exact ties.
launches. Numerics: the dot and the add accumulate in fp32, while
MarkovSampleStep rounds the bias and the sum to the step dtype; near-tie
winners can differ, so a one-row block can draft differently from the same
row in a larger batch. Drafts are proposals only (target verify guards the
output), so the impact is bounded to accept-rate noise on near ties.
"""
@classmethod
+7 -2
View File
@@ -532,6 +532,9 @@ class Envs:
SGLANG_DSPARK_ENABLE_SPS_RECORD = EnvBool(False)
SGLANG_DSPARK_FAST_KERNEL = EnvBool(True)
SGLANG_DSPARK_FP32_LM_HEAD = EnvBool(False)
# Argmax every draft row, sampling requests included, accepted as a one-hot
# draft: no step logits kept, lower acceptance at high temperatures.
SGLANG_DSPARK_USE_GREEDY_DRAFT = EnvBool(False)
SGLANG_DSPARK_FOLDED_PROPOSAL = EnvBool(True)
SGLANG_DSPARK_STACKED_CTX_KV = EnvBool(True)
SGLANG_DSPARK_EMBED_IN_GRAPH = EnvBool(True)
@@ -1979,10 +1982,12 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
note="DSpark draft sampling always uses Gumbel-max."
),
"SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV": _DeprecatedEnv(
note="Greedy DSpark draft steps take the shared markov sample step."
note="A one-row step of SGLANG_DSPARK_USE_GREEDY_DRAFT on a vanilla markov "
"head fuses the bias GEMV."
),
"SGLANG_DSPARK_FOLDED_SAMPLING": _DeprecatedEnv(
note="A folded DSpark draft proposal always samples; set "
note="A folded DSpark draft proposal samples; set "
"SGLANG_DSPARK_USE_GREEDY_DRAFT=1 for an argmax-only draft, or "
"SGLANG_DSPARK_FOLDED_PROPOSAL=0 to keep the proposal out of the graph."
),
"SGLANG_SKIP_RUST_TESTS": _DeprecatedEnv(
+17
View File
@@ -84,6 +84,11 @@ class BaseDSparkMarkovHead(nn.Module, ABC):
def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype:
"""The dtype of ``base_logits + bias``."""
def greedy_w2_weight(self) -> Optional[torch.Tensor]:
"""The w2 weight when the bias is exactly the GEMV of the previous
token's embedding, for the fused greedy kernel; otherwise None."""
return None
class VanillaMarkov(BaseDSparkMarkovHead):
def __init__(self, *, vocab_size: int, markov_rank: int) -> None:
@@ -117,6 +122,18 @@ class VanillaMarkov(BaseDSparkMarkovHead):
del hidden_states
return self.project_bias(self.get_prev_embeddings(prev_tokens)), state
def greedy_w2_weight(self) -> Optional[torch.Tensor]:
return self.markov_w2.weight if self._bias_is_w2_gemv() else None
def _bias_is_w2_gemv(self) -> bool:
# The fused kernel recomputes this class's bias; a subclass that changes
# it keeps the shared op.
cls = type(self)
return (
cls.step_bias is VanillaMarkov.step_bias
and cls.project_bias is VanillaMarkov.project_bias
)
class Nemotron35VanillaMarkov(VanillaMarkov):
"""Checkpoint-quantized Markov head used only by Nemotron 3.5 DSpark."""
@@ -42,7 +42,7 @@ def block_accept_skip_reason(
"probabilities would be state-dependent"
)
if corrected_logits is None:
return "corrected_logits unavailable (folded draft path)"
return "corrected_logits unavailable (greedy draft)"
return None
@@ -6,6 +6,9 @@ from typing import TYPE_CHECKING, Optional
import msgspec
import torch
from sglang.kernels.ops.speculative.dspark.dspark_draft_model import (
MarkovGreedyStep,
)
from sglang.kernels.ops.speculative.dspark.markov_sample import MarkovSampleStep
from sglang.srt.environ import envs
from sglang.srt.lora.layers import unwrap_lora_layer
@@ -55,6 +58,8 @@ class DraftBlockResult(msgspec.Struct, frozen=True):
corrected_logits: Optional[torch.Tensor]
greedy_mask: torch.Tensor
temperatures: torch.Tensor
# Drafted by argmax: the sampling accept uses a one-hot draft distribution.
greedy_draft: bool
class DraftForwardResult(msgspec.Struct, frozen=True):
@@ -145,6 +150,17 @@ def _draft_step(
step: int,
logits_out: Optional[torch.Tensor],
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
if logits_out is None and len(prev_tokens) == 1:
# One greedy row, no logits kept: one kernel does the GEMV, add and argmax.
w2_weight = markov_head.greedy_w2_weight()
# The kernel reads one w2 row per base-logit column.
if w2_weight is not None and w2_weight.shape[0] == base_logits.shape[-1]:
tokens = MarkovGreedyStep.execute(
base_logits=base_logits,
prev_embeds=markov_head.get_prev_embeddings(prev_tokens),
w2_weight=w2_weight,
)
return tokens, state
bias, state = markov_head.step_bias(
prev_tokens, hidden_states=hidden_states, state=state
)
@@ -184,11 +200,12 @@ def draft_markov_block(
noise: DraftNoise,
tp_sync: SpecTpSync,
logits_out: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor]:
greedy_draft: bool = False,
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
"""Returns the drafted tokens and the [bs, gamma, vocab] step logits,
written to ``logits_out`` or allocated here."""
written to ``logits_out`` or allocated here; a greedy draft keeps none."""
bs, gamma = base_logits.shape[:2]
if logits_out is None:
if not greedy_draft and logits_out is None:
logits_out = torch.empty(
(bs, gamma, step_logits_width(markov_head, base_logits.shape[-1])),
dtype=markov_head.step_logits_dtype(base_logits.dtype),
@@ -207,10 +224,12 @@ def draft_markov_block(
temperatures=step_temperatures,
noise=noise,
step=step,
logits_out=logits_out[:, step],
logits_out=None if greedy_draft else logits_out[:, step],
)
prev_tokens = tp_sync.sync(SpecTpSyncSite.DSPARK_DRAFT, prev_tokens)
draft_tokens.append(prev_tokens)
if greedy_draft:
return torch.stack(draft_tokens, dim=1), None
noise.offset.add_(1)
return torch.stack(draft_tokens, dim=1), logits_out
@@ -233,6 +252,7 @@ def sample_draft_block(
noise: DraftNoise,
device: torch.device,
tp_sync: SpecTpSync,
greedy_draft: bool,
) -> DraftBlockResult:
bs = base_logits.shape[0]
greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device)
@@ -247,12 +267,16 @@ def sample_draft_block(
step_temperatures=temperatures.masked_fill(greedy_mask, 0.0),
noise=noise,
tp_sync=tp_sync,
greedy_draft=greedy_draft,
)
return DraftBlockResult(
draft_tokens=draft_tokens,
corrected_logits=expect(_DRAFT_STEP_LOGITS, step_logits),
corrected_logits=(
None if greedy_draft else expect(_DRAFT_STEP_LOGITS, step_logits)
),
greedy_mask=greedy_mask,
temperatures=temperatures,
greedy_draft=greedy_draft,
)
@@ -267,6 +291,7 @@ class DraftBlockProposer:
draft_block_spec_info,
tp_sync: SpecTpSync,
random_seed: int,
greedy_draft: bool,
dp_moe_sync: bool = False,
) -> None:
self.draft_model = draft_model
@@ -277,6 +302,7 @@ class DraftBlockProposer:
self._mask_token_id = mask_token_id
self._draft_block_spec_info = draft_block_spec_info
self._tp_sync = tp_sync
self._greedy_draft = greedy_draft
self._noise = DraftNoise(
key=draft_noise_key(random_seed),
offset=torch.zeros(
@@ -344,9 +370,12 @@ class DraftBlockProposer:
folded = True
draft_block = DraftBlockResult(
draft_tokens=draft_sampler.out[: bs * self.gamma].view(bs, self.gamma),
corrected_logits=draft_sampler.step_logits[:bs],
corrected_logits=(
None if self._greedy_draft else draft_sampler.step_logits[:bs]
),
greedy_mask=draft_sampler.greedy_mask[:bs],
temperatures=draft_sampler.temperatures[:bs],
greedy_draft=self._greedy_draft,
)
if draft_sampler.confidence_out is not None:
folded_confidence = draft_sampler.confidence_out[:bs]
@@ -365,6 +394,7 @@ class DraftBlockProposer:
noise=self._noise,
device=device,
tp_sync=self._tp_sync,
greedy_draft=self._greedy_draft,
)
proposal_block_ids = (
draft_block_ids
@@ -28,6 +28,7 @@ class DsparkDraftSampler:
device,
noise: DraftNoise,
tp_sync: SpecTpSync,
greedy_draft: bool,
confidence_fn=None,
out=None,
):
@@ -51,6 +52,7 @@ class DsparkDraftSampler:
if confidence_fn is not None
else None
)
self.greedy_draft = greedy_draft
self._noise = noise
self._tp_sync = tp_sync
# Staged before each replay; step_temperatures is 0 on argmax rows.
@@ -62,11 +64,16 @@ class DsparkDraftSampler:
(self.max_bs,), dtype=torch.float32, device=device
)
self._zero = torch.zeros((), dtype=torch.float32, device=device)
# [max_bs, gamma, vocab] step logits the sampling accept reads.
self.step_logits = torch.empty(
(self.max_bs, self.gamma, _step_logits_width(model)),
dtype=_step_logits_dtype(model),
device=device,
# [max_bs, gamma, vocab] step logits for the sampling accept; a greedy
# draft keeps none.
self.step_logits = (
None
if greedy_draft
else torch.empty(
(self.max_bs, self.gamma, _step_logits_width(model)),
dtype=_step_logits_dtype(model),
device=device,
)
)
def stage_sampling_params(self, *, bs: int, sampling_info) -> None:
@@ -112,7 +119,8 @@ class DsparkDraftSampler:
step_temperatures=self.step_temperatures[:bs],
noise=self._noise,
tp_sync=self._tp_sync,
logits_out=self.step_logits[:bs],
logits_out=None if self.greedy_draft else self.step_logits[:bs],
greedy_draft=self.greedy_draft,
)
self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1))
if self.confidence_out is not None:
@@ -148,6 +156,7 @@ def maybe_build_draft_sampler(
device,
tp_rank: int,
noise: DraftNoise,
greedy_draft: bool,
tp_sync: SpecTpSync,
confidence_fn=None,
out=None,
@@ -165,7 +174,10 @@ def maybe_build_draft_sampler(
if draft_model.markov_head is None:
return _eager("no markov head")
if tp_rank == 0:
logger.info("DSpark draft proposal folded into the draft cuda graph.")
logger.info(
"DSpark draft proposal (%s) folded into the draft cuda graph.",
"greedy draft" if greedy_draft else "sampling",
)
return DsparkDraftSampler(
model=draft_model,
gamma=gamma,
@@ -173,6 +185,7 @@ def maybe_build_draft_sampler(
device=device,
noise=noise,
tp_sync=tp_sync,
greedy_draft=greedy_draft,
confidence_fn=confidence_fn,
out=out,
)
@@ -886,13 +886,23 @@ def accept_draft_tokens(
cutoff_verify_lens=cutoff_verify_lens,
fused_argmax=fused_argmax,
)
bs, gamma_rows, vocab = draft_block.corrected_logits.shape
draft_probs = SoftmaxTemp.execute(
logits=draft_block.corrected_logits.reshape(bs * gamma_rows, vocab),
temperatures=draft_block.temperatures,
rows_per_request=gamma_rows,
).view(bs, gamma_rows, vocab)
expect(_VERIFY_DRAFT_PROBS, draft_probs)
if draft_block.greedy_draft:
# An argmax draft is one-hot: rejection accepts at the target probability.
draft_tokens = draft_block.draft_tokens
draft_probs = torch.zeros(
(*draft_tokens.shape, target_logits.shape[-1]),
dtype=torch.float32,
device=target_logits.device,
)
draft_probs.scatter_(-1, draft_tokens.unsqueeze(-1), 1.0)
else:
bs, gamma_rows, vocab = draft_block.corrected_logits.shape
draft_probs = SoftmaxTemp.execute(
logits=draft_block.corrected_logits.reshape(bs * gamma_rows, vocab),
temperatures=draft_block.temperatures,
rows_per_request=gamma_rows,
).view(bs, gamma_rows, vocab)
expect(_VERIFY_DRAFT_PROBS, draft_probs)
if not sampling_info.is_any_greedy:
return AcceptSampling.execute(
candidates=candidates,
@@ -316,6 +316,7 @@ class DSparkWorkerV2(BaseSpecWorker):
verify_num_draft_tokens=self.verify_num_draft_tokens,
block_pos_offsets=self._block_pos_offsets,
)
self._greedy_draft = envs.SGLANG_DSPARK_USE_GREEDY_DRAFT.get()
self._proposer = DraftBlockProposer(
draft_model=self.draft_model,
draft_model_runner=self.draft_model_runner,
@@ -324,6 +325,7 @@ class DSparkWorkerV2(BaseSpecWorker):
draft_block_spec_info=self._draft_block_spec_info,
tp_sync=self._tp_sync,
random_seed=target_worker.random_seed,
greedy_draft=self._greedy_draft,
dp_moe_sync=self._draft_is_moe and get_parallel().attn_dp_enabled,
)
self._verify_epilogue = None
@@ -527,6 +529,7 @@ class DSparkWorkerV2(BaseSpecWorker):
device=self.device,
tp_rank=self._target_tp_rank,
noise=self._proposer.noise,
greedy_draft=self._greedy_draft,
tp_sync=self._tp_sync,
confidence_fn=(
self._verify_planner.compute_confidence_tensor