[DSpark] Add SGLANG_DSPARK_USE_GREEDY_DRAFT, an argmax draft accepted as one-hot (#43670)

This commit is contained in:
Liangsheng Yin
2026-10-11 15:18:59 -07:00
committed by GitHub
parent 3cc23b3b86
commit a65b776963
12 changed files with 261 additions and 30 deletions
@@ -19,11 +19,11 @@ class MarkovGreedyStep:
Computes ``argmax_v(base_logits[:, v] + dot(w2_weight[v, :], prev_embeds))``
in a single pass over the (vocab x rank) weight: no full-vocab bias or
step-logits materialization, no separate GEMV / add / two-pass argmax
launches. Numerics: the dot and the add accumulate in fp32, while the eager
path rounds the GEMV output and the add to bf16 before argmax — near-tie
winners can differ on rare steps. Drafts are proposals only (target verify
guards output correctness), so the impact is bounded to accept-rate noise
on exact ties.
launches. Numerics: the dot and the add accumulate in fp32, while
MarkovSampleStep rounds the bias and the sum to the step dtype; near-tie
winners can differ, so a one-row block can draft differently from the same
row in a larger batch. Drafts are proposals only (target verify guards the
output), so the impact is bounded to accept-rate noise on near ties.
"""
@classmethod
+7 -2
View File
@@ -532,6 +532,9 @@ class Envs:
SGLANG_DSPARK_ENABLE_SPS_RECORD = EnvBool(False)
SGLANG_DSPARK_FAST_KERNEL = EnvBool(True)
SGLANG_DSPARK_FP32_LM_HEAD = EnvBool(False)
# Argmax every draft row, sampling requests included, accepted as a one-hot
# draft: no step logits kept, lower acceptance at high temperatures.
SGLANG_DSPARK_USE_GREEDY_DRAFT = EnvBool(False)
SGLANG_DSPARK_FOLDED_PROPOSAL = EnvBool(True)
SGLANG_DSPARK_STACKED_CTX_KV = EnvBool(True)
SGLANG_DSPARK_EMBED_IN_GRAPH = EnvBool(True)
@@ -1979,10 +1982,12 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
note="DSpark draft sampling always uses Gumbel-max."
),
"SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV": _DeprecatedEnv(
note="Greedy DSpark draft steps take the shared markov sample step."
note="A one-row step of SGLANG_DSPARK_USE_GREEDY_DRAFT on a vanilla markov "
"head fuses the bias GEMV."
),
"SGLANG_DSPARK_FOLDED_SAMPLING": _DeprecatedEnv(
note="A folded DSpark draft proposal always samples; set "
note="A folded DSpark draft proposal samples; set "
"SGLANG_DSPARK_USE_GREEDY_DRAFT=1 for an argmax-only draft, or "
"SGLANG_DSPARK_FOLDED_PROPOSAL=0 to keep the proposal out of the graph."
),
"SGLANG_SKIP_RUST_TESTS": _DeprecatedEnv(
+17
View File
@@ -84,6 +84,11 @@ class BaseDSparkMarkovHead(nn.Module, ABC):
def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype:
"""The dtype of ``base_logits + bias``."""
def greedy_w2_weight(self) -> Optional[torch.Tensor]:
"""The w2 weight when the bias is exactly the GEMV of the previous
token's embedding, for the fused greedy kernel; otherwise None."""
return None
class VanillaMarkov(BaseDSparkMarkovHead):
def __init__(self, *, vocab_size: int, markov_rank: int) -> None:
@@ -117,6 +122,18 @@ class VanillaMarkov(BaseDSparkMarkovHead):
del hidden_states
return self.project_bias(self.get_prev_embeddings(prev_tokens)), state
def greedy_w2_weight(self) -> Optional[torch.Tensor]:
return self.markov_w2.weight if self._bias_is_w2_gemv() else None
def _bias_is_w2_gemv(self) -> bool:
# The fused kernel recomputes this class's bias; a subclass that changes
# it keeps the shared op.
cls = type(self)
return (
cls.step_bias is VanillaMarkov.step_bias
and cls.project_bias is VanillaMarkov.project_bias
)
class Nemotron35VanillaMarkov(VanillaMarkov):
"""Checkpoint-quantized Markov head used only by Nemotron 3.5 DSpark."""
@@ -42,7 +42,7 @@ def block_accept_skip_reason(
"probabilities would be state-dependent"
)
if corrected_logits is None:
return "corrected_logits unavailable (folded draft path)"
return "corrected_logits unavailable (greedy draft)"
return None
@@ -6,6 +6,9 @@ from typing import TYPE_CHECKING, Optional
import msgspec
import torch
from sglang.kernels.ops.speculative.dspark.dspark_draft_model import (
MarkovGreedyStep,
)
from sglang.kernels.ops.speculative.dspark.markov_sample import MarkovSampleStep
from sglang.srt.environ import envs
from sglang.srt.lora.layers import unwrap_lora_layer
@@ -55,6 +58,8 @@ class DraftBlockResult(msgspec.Struct, frozen=True):
corrected_logits: Optional[torch.Tensor]
greedy_mask: torch.Tensor
temperatures: torch.Tensor
# Drafted by argmax: the sampling accept uses a one-hot draft distribution.
greedy_draft: bool
class DraftForwardResult(msgspec.Struct, frozen=True):
@@ -145,6 +150,17 @@ def _draft_step(
step: int,
logits_out: Optional[torch.Tensor],
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
if logits_out is None and len(prev_tokens) == 1:
# One greedy row, no logits kept: one kernel does the GEMV, add and argmax.
w2_weight = markov_head.greedy_w2_weight()
# The kernel reads one w2 row per base-logit column.
if w2_weight is not None and w2_weight.shape[0] == base_logits.shape[-1]:
tokens = MarkovGreedyStep.execute(
base_logits=base_logits,
prev_embeds=markov_head.get_prev_embeddings(prev_tokens),
w2_weight=w2_weight,
)
return tokens, state
bias, state = markov_head.step_bias(
prev_tokens, hidden_states=hidden_states, state=state
)
@@ -184,11 +200,12 @@ def draft_markov_block(
noise: DraftNoise,
tp_sync: SpecTpSync,
logits_out: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor]:
greedy_draft: bool = False,
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
"""Returns the drafted tokens and the [bs, gamma, vocab] step logits,
written to ``logits_out`` or allocated here."""
written to ``logits_out`` or allocated here; a greedy draft keeps none."""
bs, gamma = base_logits.shape[:2]
if logits_out is None:
if not greedy_draft and logits_out is None:
logits_out = torch.empty(
(bs, gamma, step_logits_width(markov_head, base_logits.shape[-1])),
dtype=markov_head.step_logits_dtype(base_logits.dtype),
@@ -207,10 +224,12 @@ def draft_markov_block(
temperatures=step_temperatures,
noise=noise,
step=step,
logits_out=logits_out[:, step],
logits_out=None if greedy_draft else logits_out[:, step],
)
prev_tokens = tp_sync.sync(SpecTpSyncSite.DSPARK_DRAFT, prev_tokens)
draft_tokens.append(prev_tokens)
if greedy_draft:
return torch.stack(draft_tokens, dim=1), None
noise.offset.add_(1)
return torch.stack(draft_tokens, dim=1), logits_out
@@ -233,6 +252,7 @@ def sample_draft_block(
noise: DraftNoise,
device: torch.device,
tp_sync: SpecTpSync,
greedy_draft: bool,
) -> DraftBlockResult:
bs = base_logits.shape[0]
greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device)
@@ -247,12 +267,16 @@ def sample_draft_block(
step_temperatures=temperatures.masked_fill(greedy_mask, 0.0),
noise=noise,
tp_sync=tp_sync,
greedy_draft=greedy_draft,
)
return DraftBlockResult(
draft_tokens=draft_tokens,
corrected_logits=expect(_DRAFT_STEP_LOGITS, step_logits),
corrected_logits=(
None if greedy_draft else expect(_DRAFT_STEP_LOGITS, step_logits)
),
greedy_mask=greedy_mask,
temperatures=temperatures,
greedy_draft=greedy_draft,
)
@@ -267,6 +291,7 @@ class DraftBlockProposer:
draft_block_spec_info,
tp_sync: SpecTpSync,
random_seed: int,
greedy_draft: bool,
dp_moe_sync: bool = False,
) -> None:
self.draft_model = draft_model
@@ -277,6 +302,7 @@ class DraftBlockProposer:
self._mask_token_id = mask_token_id
self._draft_block_spec_info = draft_block_spec_info
self._tp_sync = tp_sync
self._greedy_draft = greedy_draft
self._noise = DraftNoise(
key=draft_noise_key(random_seed),
offset=torch.zeros(
@@ -344,9 +370,12 @@ class DraftBlockProposer:
folded = True
draft_block = DraftBlockResult(
draft_tokens=draft_sampler.out[: bs * self.gamma].view(bs, self.gamma),
corrected_logits=draft_sampler.step_logits[:bs],
corrected_logits=(
None if self._greedy_draft else draft_sampler.step_logits[:bs]
),
greedy_mask=draft_sampler.greedy_mask[:bs],
temperatures=draft_sampler.temperatures[:bs],
greedy_draft=self._greedy_draft,
)
if draft_sampler.confidence_out is not None:
folded_confidence = draft_sampler.confidence_out[:bs]
@@ -365,6 +394,7 @@ class DraftBlockProposer:
noise=self._noise,
device=device,
tp_sync=self._tp_sync,
greedy_draft=self._greedy_draft,
)
proposal_block_ids = (
draft_block_ids
@@ -28,6 +28,7 @@ class DsparkDraftSampler:
device,
noise: DraftNoise,
tp_sync: SpecTpSync,
greedy_draft: bool,
confidence_fn=None,
out=None,
):
@@ -51,6 +52,7 @@ class DsparkDraftSampler:
if confidence_fn is not None
else None
)
self.greedy_draft = greedy_draft
self._noise = noise
self._tp_sync = tp_sync
# Staged before each replay; step_temperatures is 0 on argmax rows.
@@ -62,11 +64,16 @@ class DsparkDraftSampler:
(self.max_bs,), dtype=torch.float32, device=device
)
self._zero = torch.zeros((), dtype=torch.float32, device=device)
# [max_bs, gamma, vocab] step logits the sampling accept reads.
self.step_logits = torch.empty(
(self.max_bs, self.gamma, _step_logits_width(model)),
dtype=_step_logits_dtype(model),
device=device,
# [max_bs, gamma, vocab] step logits for the sampling accept; a greedy
# draft keeps none.
self.step_logits = (
None
if greedy_draft
else torch.empty(
(self.max_bs, self.gamma, _step_logits_width(model)),
dtype=_step_logits_dtype(model),
device=device,
)
)
def stage_sampling_params(self, *, bs: int, sampling_info) -> None:
@@ -112,7 +119,8 @@ class DsparkDraftSampler:
step_temperatures=self.step_temperatures[:bs],
noise=self._noise,
tp_sync=self._tp_sync,
logits_out=self.step_logits[:bs],
logits_out=None if self.greedy_draft else self.step_logits[:bs],
greedy_draft=self.greedy_draft,
)
self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1))
if self.confidence_out is not None:
@@ -148,6 +156,7 @@ def maybe_build_draft_sampler(
device,
tp_rank: int,
noise: DraftNoise,
greedy_draft: bool,
tp_sync: SpecTpSync,
confidence_fn=None,
out=None,
@@ -165,7 +174,10 @@ def maybe_build_draft_sampler(
if draft_model.markov_head is None:
return _eager("no markov head")
if tp_rank == 0:
logger.info("DSpark draft proposal folded into the draft cuda graph.")
logger.info(
"DSpark draft proposal (%s) folded into the draft cuda graph.",
"greedy draft" if greedy_draft else "sampling",
)
return DsparkDraftSampler(
model=draft_model,
gamma=gamma,
@@ -173,6 +185,7 @@ def maybe_build_draft_sampler(
device=device,
noise=noise,
tp_sync=tp_sync,
greedy_draft=greedy_draft,
confidence_fn=confidence_fn,
out=out,
)
@@ -886,13 +886,23 @@ def accept_draft_tokens(
cutoff_verify_lens=cutoff_verify_lens,
fused_argmax=fused_argmax,
)
bs, gamma_rows, vocab = draft_block.corrected_logits.shape
draft_probs = SoftmaxTemp.execute(
logits=draft_block.corrected_logits.reshape(bs * gamma_rows, vocab),
temperatures=draft_block.temperatures,
rows_per_request=gamma_rows,
).view(bs, gamma_rows, vocab)
expect(_VERIFY_DRAFT_PROBS, draft_probs)
if draft_block.greedy_draft:
# An argmax draft is one-hot: rejection accepts at the target probability.
draft_tokens = draft_block.draft_tokens
draft_probs = torch.zeros(
(*draft_tokens.shape, target_logits.shape[-1]),
dtype=torch.float32,
device=target_logits.device,
)
draft_probs.scatter_(-1, draft_tokens.unsqueeze(-1), 1.0)
else:
bs, gamma_rows, vocab = draft_block.corrected_logits.shape
draft_probs = SoftmaxTemp.execute(
logits=draft_block.corrected_logits.reshape(bs * gamma_rows, vocab),
temperatures=draft_block.temperatures,
rows_per_request=gamma_rows,
).view(bs, gamma_rows, vocab)
expect(_VERIFY_DRAFT_PROBS, draft_probs)
if not sampling_info.is_any_greedy:
return AcceptSampling.execute(
candidates=candidates,
@@ -316,6 +316,7 @@ class DSparkWorkerV2(BaseSpecWorker):
verify_num_draft_tokens=self.verify_num_draft_tokens,
block_pos_offsets=self._block_pos_offsets,
)
self._greedy_draft = envs.SGLANG_DSPARK_USE_GREEDY_DRAFT.get()
self._proposer = DraftBlockProposer(
draft_model=self.draft_model,
draft_model_runner=self.draft_model_runner,
@@ -324,6 +325,7 @@ class DSparkWorkerV2(BaseSpecWorker):
draft_block_spec_info=self._draft_block_spec_info,
tp_sync=self._tp_sync,
random_seed=target_worker.random_seed,
greedy_draft=self._greedy_draft,
dp_moe_sync=self._draft_is_moe and get_parallel().attn_dp_enabled,
)
self._verify_epilogue = None
@@ -527,6 +529,7 @@ class DSparkWorkerV2(BaseSpecWorker):
device=self.device,
tp_rank=self._target_tp_rank,
noise=self._proposer.noise,
greedy_draft=self._greedy_draft,
tp_sync=self._tp_sync,
confidence_fn=(
self._verify_planner.compute_confidence_tensor
@@ -360,6 +360,7 @@ def test_dflash_family_accept_uses_block_verification(any_greedy):
corrected_logits=draft_logits,
greedy_mask=greedy_mask,
temperatures=torch.ones(batch, device=DEVICE),
greedy_draft=False,
),
sampling_info=SimpleNamespace(
is_all_greedy=False,
@@ -1,8 +1,12 @@
import unittest
from types import SimpleNamespace
from unittest import mock
import torch
from sglang.kernels.ops.speculative.dspark.dspark_draft_model import (
MarkovGreedyStep,
)
from sglang.srt.models.dspark import VanillaMarkov
from sglang.srt.speculative.dspark_components.dspark_draft import (
DraftNoise,
@@ -69,6 +73,52 @@ class TestDraftMarkovBlockGraph(CustomTestCase):
self.assertFalse(torch.equal(tokens_out[1::2], eager_tokens[1::2]))
self.assertTrue(torch.equal(tokens_out[::2], eager_tokens[::2]))
def test_one_greedy_row_matches_eager_and_fp32_reference(self):
# The fused one-row kernel runs; the graph replays eager; both match FP32.
torch.manual_seed(2)
head = VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK).to(DEVICE)
head = head.to(torch.bfloat16)
base = torch.randn(1, GAMMA, VOCAB, device=DEVICE).to(torch.bfloat16)
anchor = torch.randint(0, VOCAB, (1,), device=DEVICE)
noise = DraftNoise(
key=3, offset=torch.zeros(1, dtype=torch.int64, device=DEVICE)
)
tokens_out = torch.empty(1, GAMMA, dtype=torch.int64, device=DEVICE)
def block():
tokens, logits = draft_markov_block(
markov_head=head,
base_logits=base,
anchor_tokens=anchor,
draft_hidden=None,
step_temperatures=torch.zeros(1, device=DEVICE),
noise=noise,
tp_sync=SimpleNamespace(sync=lambda site, value: value),
greedy_draft=True,
)
assert logits is None
tokens_out.copy_(tokens)
with mock.patch.object(
MarkovGreedyStep, "execute", wraps=MarkovGreedyStep.execute
) as fused:
block()
self.assertEqual(fused.call_count, GAMMA)
eager_tokens = tokens_out.clone()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
block()
tokens_out.zero_()
graph.replay()
torch.cuda.synchronize()
self.assertTrue(torch.equal(tokens_out, eager_tokens))
prev = anchor
for step in range(GAMMA):
bias = head.markov_w1(prev).float() @ head.markov_w2.weight.float().T
prev = (base[:, step].float() + bias).argmax(-1)
self.assertEqual(eager_tokens[0, step].item(), prev.item())
def test_captured_sampler_matches_eager_block(self):
# The real tail (staging, static buffers, [:bs] views) against eager.
torch.manual_seed(1)
@@ -92,6 +142,7 @@ class TestDraftMarkovBlockGraph(CustomTestCase):
device=DEVICE,
noise=noise,
tp_sync=tp_sync,
greedy_draft=False,
)
hidden = torch.randn(BS * GAMMA, 64, device=DEVICE).to(torch.bfloat16)
input_ids = torch.randint(0, VOCAB, (BS * GAMMA,), device=DEVICE)
@@ -121,6 +172,7 @@ class TestDraftMarkovBlockGraph(CustomTestCase):
noise=eager_noise,
device=DEVICE,
tp_sync=tp_sync,
greedy_draft=False,
)
self.assertTrue(
torch.equal(sampler.out[: BS * GAMMA].view(BS, GAMMA), eager.draft_tokens)
@@ -3,7 +3,12 @@ from types import SimpleNamespace
import torch
from sglang.srt.models.dspark import RNNHead, VanillaMarkov
from sglang.srt.models.dspark import (
GatedMarkovHead,
Nemotron35VanillaMarkov,
RNNHead,
VanillaMarkov,
)
from sglang.srt.speculative.dspark_components.dspark_draft import (
DraftNoise,
sample_draft_block,
@@ -16,7 +21,7 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu")
VOCAB, RANK, HIDDEN, BS, GAMMA = 211, 8, 16, 4, 5
def _draft(head, sampling_info, offset):
def _draft(head, sampling_info, offset, greedy_draft=False):
torch.manual_seed(0)
base = torch.randn(BS, GAMMA, VOCAB).to(torch.bfloat16)
hidden = torch.randn(BS, GAMMA, HIDDEN).to(torch.bfloat16)
@@ -30,6 +35,7 @@ def _draft(head, sampling_info, offset):
noise=DraftNoise(key=3, offset=offset),
device="cpu",
tp_sync=SimpleNamespace(sync=lambda site, tokens: tokens),
greedy_draft=greedy_draft,
)
return block, base, hidden, anchor
@@ -78,6 +84,45 @@ class TestSampleDraftBlock(CustomTestCase):
self.assertEqual(block.greedy_mask.tolist(), greedy.tolist())
self.assertEqual(offset.item(), 1)
def test_greedy_draft_takes_every_argmax_and_keeps_no_logits(self):
head = VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK).to(torch.bfloat16)
# Sampling requests too: a greedy draft ignores their temperatures.
sampling_info = SimpleNamespace(
is_all_greedy=False,
temperatures=torch.tensor([[0.7], [1.0], [1.0], [1.3]]),
top_ks=torch.tensor([50, 50, 1, 50]),
)
offset = torch.zeros(1, dtype=torch.int64)
block, base, hidden, anchor = _draft(
head, sampling_info, offset, greedy_draft=True
)
logits = _step_logits_along(head, base, hidden, anchor, block.draft_tokens)
self.assertTrue(torch.equal(block.draft_tokens, logits.float().argmax(-1)))
self.assertIsNone(block.corrected_logits)
self.assertTrue(block.greedy_draft)
class TestFusedGreedyEligibility(CustomTestCase):
def test_only_the_plain_w2_gemv_head_fuses(self):
# A head that changes the bias must keep the shared op.
heads = (
(VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK), True),
(
Nemotron35VanillaMarkov(
vocab_size=VOCAB, markov_rank=RANK, quant_config=None, prefix=""
),
False,
),
(
GatedMarkovHead(vocab_size=VOCAB, markov_rank=RANK, hidden_size=HIDDEN),
False,
),
(RNNHead(vocab_size=VOCAB, markov_rank=RANK, hidden_size=HIDDEN), False),
)
for head, fuses in heads:
with self.subTest(head=type(head).__name__):
self.assertIs(head.greedy_w2_weight() is not None, fuses)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,55 @@
import unittest
from types import SimpleNamespace
from unittest import mock
import torch
from sglang.srt.speculative.dspark_components import dspark_verify
from sglang.srt.speculative.dspark_components.dspark_draft import DraftBlockResult
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
BS, GAMMA, VOCAB = 3, 4, 50
class TestGreedyDraftAccept(CustomTestCase):
def test_sampling_accept_reads_a_one_hot_draft(self):
# The sampling accept gets a one-hot draft at the target vocab width.
torch.manual_seed(0)
draft_tokens = torch.randint(0, VOCAB, (BS, GAMMA))
block = DraftBlockResult(
draft_tokens=draft_tokens,
corrected_logits=None,
greedy_mask=torch.zeros(BS, dtype=torch.bool),
temperatures=torch.ones(BS),
greedy_draft=True,
)
seen = {}
def fake_accept(**kwargs):
seen["draft_probs"] = kwargs["draft_probs"]
zeros = torch.zeros(BS, dtype=torch.int64)
return zeros, zeros, zeros
with mock.patch.object(
dspark_verify.AcceptSampling, "execute", side_effect=fake_accept
):
dspark_verify.accept_draft_tokens(
candidates=torch.zeros(BS, GAMMA + 1, dtype=torch.int64),
target_logits=torch.randn(BS * (GAMMA + 1), VOCAB),
draft_block=block,
sampling_info=SimpleNamespace(is_all_greedy=False, is_any_greedy=False),
draft_input=None,
gamma=GAMMA,
verify_num_draft_tokens=GAMMA + 1,
)
probs = seen["draft_probs"]
self.assertEqual(probs.shape, (BS, GAMMA, VOCAB))
expected = torch.nn.functional.one_hot(draft_tokens, VOCAB).float()
self.assertTrue(torch.equal(probs, expected))
if __name__ == "__main__":
unittest.main()