From a65b776963d3e9139a82ac38fced030d0bbb6acf Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sun, 11 Oct 2026 15:18:59 -0700 Subject: [PATCH] [DSpark] Add `SGLANG_DSPARK_USE_GREEDY_DRAFT`, an argmax draft accepted as one-hot (#43670) --- .../speculative/dspark/dspark_draft_model.py | 10 ++-- python/sglang/srt/environ.py | 9 ++- python/sglang/srt/models/dspark.py | 17 ++++++ .../dspark_block_accept_estimator.py | 2 +- .../dspark_components/dspark_draft.py | 42 ++++++++++++-- .../dspark_components/dspark_draft_sampler.py | 27 ++++++--- .../dspark_components/dspark_verify.py | 24 +++++--- .../dspark_components/dspark_worker_v2.py | 3 + .../speculative/test_block_verification.py | 1 + .../spec/dspark/test_dspark_draft_graph.py | 52 ++++++++++++++++++ .../unit/spec/test_dspark_draft_block.py | 49 ++++++++++++++++- .../spec/test_dspark_greedy_draft_accept.py | 55 +++++++++++++++++++ 12 files changed, 261 insertions(+), 30 deletions(-) create mode 100644 test/registered/unit/spec/test_dspark_greedy_draft_accept.py diff --git a/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py b/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py index 5d295b7ec0..2890366ac2 100644 --- a/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py +++ b/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py @@ -19,11 +19,11 @@ class MarkovGreedyStep: Computes ``argmax_v(base_logits[:, v] + dot(w2_weight[v, :], prev_embeds))`` in a single pass over the (vocab x rank) weight: no full-vocab bias or step-logits materialization, no separate GEMV / add / two-pass argmax - launches. Numerics: the dot and the add accumulate in fp32, while the eager - path rounds the GEMV output and the add to bf16 before argmax — near-tie - winners can differ on rare steps. Drafts are proposals only (target verify - guards output correctness), so the impact is bounded to accept-rate noise - on exact ties. + launches. Numerics: the dot and the add accumulate in fp32, while + MarkovSampleStep rounds the bias and the sum to the step dtype; near-tie + winners can differ, so a one-row block can draft differently from the same + row in a larger batch. Drafts are proposals only (target verify guards the + output), so the impact is bounded to accept-rate noise on near ties. """ @classmethod diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index a2dd2f1248..150648516f 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -532,6 +532,9 @@ class Envs: SGLANG_DSPARK_ENABLE_SPS_RECORD = EnvBool(False) SGLANG_DSPARK_FAST_KERNEL = EnvBool(True) SGLANG_DSPARK_FP32_LM_HEAD = EnvBool(False) + # Argmax every draft row, sampling requests included, accepted as a one-hot + # draft: no step logits kept, lower acceptance at high temperatures. + SGLANG_DSPARK_USE_GREEDY_DRAFT = EnvBool(False) SGLANG_DSPARK_FOLDED_PROPOSAL = EnvBool(True) SGLANG_DSPARK_STACKED_CTX_KV = EnvBool(True) SGLANG_DSPARK_EMBED_IN_GRAPH = EnvBool(True) @@ -1979,10 +1982,12 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = { note="DSpark draft sampling always uses Gumbel-max." ), "SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV": _DeprecatedEnv( - note="Greedy DSpark draft steps take the shared markov sample step." + note="A one-row step of SGLANG_DSPARK_USE_GREEDY_DRAFT on a vanilla markov " + "head fuses the bias GEMV." ), "SGLANG_DSPARK_FOLDED_SAMPLING": _DeprecatedEnv( - note="A folded DSpark draft proposal always samples; set " + note="A folded DSpark draft proposal samples; set " + "SGLANG_DSPARK_USE_GREEDY_DRAFT=1 for an argmax-only draft, or " "SGLANG_DSPARK_FOLDED_PROPOSAL=0 to keep the proposal out of the graph." ), "SGLANG_SKIP_RUST_TESTS": _DeprecatedEnv( diff --git a/python/sglang/srt/models/dspark.py b/python/sglang/srt/models/dspark.py index 566d39f58a..000c6bed9e 100644 --- a/python/sglang/srt/models/dspark.py +++ b/python/sglang/srt/models/dspark.py @@ -84,6 +84,11 @@ class BaseDSparkMarkovHead(nn.Module, ABC): def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype: """The dtype of ``base_logits + bias``.""" + def greedy_w2_weight(self) -> Optional[torch.Tensor]: + """The w2 weight when the bias is exactly the GEMV of the previous + token's embedding, for the fused greedy kernel; otherwise None.""" + return None + class VanillaMarkov(BaseDSparkMarkovHead): def __init__(self, *, vocab_size: int, markov_rank: int) -> None: @@ -117,6 +122,18 @@ class VanillaMarkov(BaseDSparkMarkovHead): del hidden_states return self.project_bias(self.get_prev_embeddings(prev_tokens)), state + def greedy_w2_weight(self) -> Optional[torch.Tensor]: + return self.markov_w2.weight if self._bias_is_w2_gemv() else None + + def _bias_is_w2_gemv(self) -> bool: + # The fused kernel recomputes this class's bias; a subclass that changes + # it keeps the shared op. + cls = type(self) + return ( + cls.step_bias is VanillaMarkov.step_bias + and cls.project_bias is VanillaMarkov.project_bias + ) + class Nemotron35VanillaMarkov(VanillaMarkov): """Checkpoint-quantized Markov head used only by Nemotron 3.5 DSpark.""" diff --git a/python/sglang/srt/speculative/dspark_components/dspark_block_accept_estimator.py b/python/sglang/srt/speculative/dspark_components/dspark_block_accept_estimator.py index 0a77646d79..831c21ed44 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_block_accept_estimator.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_block_accept_estimator.py @@ -42,7 +42,7 @@ def block_accept_skip_reason( "probabilities would be state-dependent" ) if corrected_logits is None: - return "corrected_logits unavailable (folded draft path)" + return "corrected_logits unavailable (greedy draft)" return None diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft.py b/python/sglang/srt/speculative/dspark_components/dspark_draft.py index 9132776636..eebd6d89bc 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft.py @@ -6,6 +6,9 @@ from typing import TYPE_CHECKING, Optional import msgspec import torch +from sglang.kernels.ops.speculative.dspark.dspark_draft_model import ( + MarkovGreedyStep, +) from sglang.kernels.ops.speculative.dspark.markov_sample import MarkovSampleStep from sglang.srt.environ import envs from sglang.srt.lora.layers import unwrap_lora_layer @@ -55,6 +58,8 @@ class DraftBlockResult(msgspec.Struct, frozen=True): corrected_logits: Optional[torch.Tensor] greedy_mask: torch.Tensor temperatures: torch.Tensor + # Drafted by argmax: the sampling accept uses a one-hot draft distribution. + greedy_draft: bool class DraftForwardResult(msgspec.Struct, frozen=True): @@ -145,6 +150,17 @@ def _draft_step( step: int, logits_out: Optional[torch.Tensor], ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + if logits_out is None and len(prev_tokens) == 1: + # One greedy row, no logits kept: one kernel does the GEMV, add and argmax. + w2_weight = markov_head.greedy_w2_weight() + # The kernel reads one w2 row per base-logit column. + if w2_weight is not None and w2_weight.shape[0] == base_logits.shape[-1]: + tokens = MarkovGreedyStep.execute( + base_logits=base_logits, + prev_embeds=markov_head.get_prev_embeddings(prev_tokens), + w2_weight=w2_weight, + ) + return tokens, state bias, state = markov_head.step_bias( prev_tokens, hidden_states=hidden_states, state=state ) @@ -184,11 +200,12 @@ def draft_markov_block( noise: DraftNoise, tp_sync: SpecTpSync, logits_out: Optional[torch.Tensor] = None, -) -> tuple[torch.Tensor, torch.Tensor]: + greedy_draft: bool = False, +) -> tuple[torch.Tensor, Optional[torch.Tensor]]: """Returns the drafted tokens and the [bs, gamma, vocab] step logits, - written to ``logits_out`` or allocated here.""" + written to ``logits_out`` or allocated here; a greedy draft keeps none.""" bs, gamma = base_logits.shape[:2] - if logits_out is None: + if not greedy_draft and logits_out is None: logits_out = torch.empty( (bs, gamma, step_logits_width(markov_head, base_logits.shape[-1])), dtype=markov_head.step_logits_dtype(base_logits.dtype), @@ -207,10 +224,12 @@ def draft_markov_block( temperatures=step_temperatures, noise=noise, step=step, - logits_out=logits_out[:, step], + logits_out=None if greedy_draft else logits_out[:, step], ) prev_tokens = tp_sync.sync(SpecTpSyncSite.DSPARK_DRAFT, prev_tokens) draft_tokens.append(prev_tokens) + if greedy_draft: + return torch.stack(draft_tokens, dim=1), None noise.offset.add_(1) return torch.stack(draft_tokens, dim=1), logits_out @@ -233,6 +252,7 @@ def sample_draft_block( noise: DraftNoise, device: torch.device, tp_sync: SpecTpSync, + greedy_draft: bool, ) -> DraftBlockResult: bs = base_logits.shape[0] greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device) @@ -247,12 +267,16 @@ def sample_draft_block( step_temperatures=temperatures.masked_fill(greedy_mask, 0.0), noise=noise, tp_sync=tp_sync, + greedy_draft=greedy_draft, ) return DraftBlockResult( draft_tokens=draft_tokens, - corrected_logits=expect(_DRAFT_STEP_LOGITS, step_logits), + corrected_logits=( + None if greedy_draft else expect(_DRAFT_STEP_LOGITS, step_logits) + ), greedy_mask=greedy_mask, temperatures=temperatures, + greedy_draft=greedy_draft, ) @@ -267,6 +291,7 @@ class DraftBlockProposer: draft_block_spec_info, tp_sync: SpecTpSync, random_seed: int, + greedy_draft: bool, dp_moe_sync: bool = False, ) -> None: self.draft_model = draft_model @@ -277,6 +302,7 @@ class DraftBlockProposer: self._mask_token_id = mask_token_id self._draft_block_spec_info = draft_block_spec_info self._tp_sync = tp_sync + self._greedy_draft = greedy_draft self._noise = DraftNoise( key=draft_noise_key(random_seed), offset=torch.zeros( @@ -344,9 +370,12 @@ class DraftBlockProposer: folded = True draft_block = DraftBlockResult( draft_tokens=draft_sampler.out[: bs * self.gamma].view(bs, self.gamma), - corrected_logits=draft_sampler.step_logits[:bs], + corrected_logits=( + None if self._greedy_draft else draft_sampler.step_logits[:bs] + ), greedy_mask=draft_sampler.greedy_mask[:bs], temperatures=draft_sampler.temperatures[:bs], + greedy_draft=self._greedy_draft, ) if draft_sampler.confidence_out is not None: folded_confidence = draft_sampler.confidence_out[:bs] @@ -365,6 +394,7 @@ class DraftBlockProposer: noise=self._noise, device=device, tp_sync=self._tp_sync, + greedy_draft=self._greedy_draft, ) proposal_block_ids = ( draft_block_ids diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py index 4332bae31b..64f1f9160c 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py @@ -28,6 +28,7 @@ class DsparkDraftSampler: device, noise: DraftNoise, tp_sync: SpecTpSync, + greedy_draft: bool, confidence_fn=None, out=None, ): @@ -51,6 +52,7 @@ class DsparkDraftSampler: if confidence_fn is not None else None ) + self.greedy_draft = greedy_draft self._noise = noise self._tp_sync = tp_sync # Staged before each replay; step_temperatures is 0 on argmax rows. @@ -62,11 +64,16 @@ class DsparkDraftSampler: (self.max_bs,), dtype=torch.float32, device=device ) self._zero = torch.zeros((), dtype=torch.float32, device=device) - # [max_bs, gamma, vocab] step logits the sampling accept reads. - self.step_logits = torch.empty( - (self.max_bs, self.gamma, _step_logits_width(model)), - dtype=_step_logits_dtype(model), - device=device, + # [max_bs, gamma, vocab] step logits for the sampling accept; a greedy + # draft keeps none. + self.step_logits = ( + None + if greedy_draft + else torch.empty( + (self.max_bs, self.gamma, _step_logits_width(model)), + dtype=_step_logits_dtype(model), + device=device, + ) ) def stage_sampling_params(self, *, bs: int, sampling_info) -> None: @@ -112,7 +119,8 @@ class DsparkDraftSampler: step_temperatures=self.step_temperatures[:bs], noise=self._noise, tp_sync=self._tp_sync, - logits_out=self.step_logits[:bs], + logits_out=None if self.greedy_draft else self.step_logits[:bs], + greedy_draft=self.greedy_draft, ) self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1)) if self.confidence_out is not None: @@ -148,6 +156,7 @@ def maybe_build_draft_sampler( device, tp_rank: int, noise: DraftNoise, + greedy_draft: bool, tp_sync: SpecTpSync, confidence_fn=None, out=None, @@ -165,7 +174,10 @@ def maybe_build_draft_sampler( if draft_model.markov_head is None: return _eager("no markov head") if tp_rank == 0: - logger.info("DSpark draft proposal folded into the draft cuda graph.") + logger.info( + "DSpark draft proposal (%s) folded into the draft cuda graph.", + "greedy draft" if greedy_draft else "sampling", + ) return DsparkDraftSampler( model=draft_model, gamma=gamma, @@ -173,6 +185,7 @@ def maybe_build_draft_sampler( device=device, noise=noise, tp_sync=tp_sync, + greedy_draft=greedy_draft, confidence_fn=confidence_fn, out=out, ) diff --git a/python/sglang/srt/speculative/dspark_components/dspark_verify.py b/python/sglang/srt/speculative/dspark_components/dspark_verify.py index 3eb1495592..c25230cc62 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_verify.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_verify.py @@ -886,13 +886,23 @@ def accept_draft_tokens( cutoff_verify_lens=cutoff_verify_lens, fused_argmax=fused_argmax, ) - bs, gamma_rows, vocab = draft_block.corrected_logits.shape - draft_probs = SoftmaxTemp.execute( - logits=draft_block.corrected_logits.reshape(bs * gamma_rows, vocab), - temperatures=draft_block.temperatures, - rows_per_request=gamma_rows, - ).view(bs, gamma_rows, vocab) - expect(_VERIFY_DRAFT_PROBS, draft_probs) + if draft_block.greedy_draft: + # An argmax draft is one-hot: rejection accepts at the target probability. + draft_tokens = draft_block.draft_tokens + draft_probs = torch.zeros( + (*draft_tokens.shape, target_logits.shape[-1]), + dtype=torch.float32, + device=target_logits.device, + ) + draft_probs.scatter_(-1, draft_tokens.unsqueeze(-1), 1.0) + else: + bs, gamma_rows, vocab = draft_block.corrected_logits.shape + draft_probs = SoftmaxTemp.execute( + logits=draft_block.corrected_logits.reshape(bs * gamma_rows, vocab), + temperatures=draft_block.temperatures, + rows_per_request=gamma_rows, + ).view(bs, gamma_rows, vocab) + expect(_VERIFY_DRAFT_PROBS, draft_probs) if not sampling_info.is_any_greedy: return AcceptSampling.execute( candidates=candidates, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index 78cedf0380..843feb839f 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -316,6 +316,7 @@ class DSparkWorkerV2(BaseSpecWorker): verify_num_draft_tokens=self.verify_num_draft_tokens, block_pos_offsets=self._block_pos_offsets, ) + self._greedy_draft = envs.SGLANG_DSPARK_USE_GREEDY_DRAFT.get() self._proposer = DraftBlockProposer( draft_model=self.draft_model, draft_model_runner=self.draft_model_runner, @@ -324,6 +325,7 @@ class DSparkWorkerV2(BaseSpecWorker): draft_block_spec_info=self._draft_block_spec_info, tp_sync=self._tp_sync, random_seed=target_worker.random_seed, + greedy_draft=self._greedy_draft, dp_moe_sync=self._draft_is_moe and get_parallel().attn_dp_enabled, ) self._verify_epilogue = None @@ -527,6 +529,7 @@ class DSparkWorkerV2(BaseSpecWorker): device=self.device, tp_rank=self._target_tp_rank, noise=self._proposer.noise, + greedy_draft=self._greedy_draft, tp_sync=self._tp_sync, confidence_fn=( self._verify_planner.compute_confidence_tensor diff --git a/test/registered/kernels/ops/speculative/test_block_verification.py b/test/registered/kernels/ops/speculative/test_block_verification.py index 2e69343d7b..f824f21d96 100644 --- a/test/registered/kernels/ops/speculative/test_block_verification.py +++ b/test/registered/kernels/ops/speculative/test_block_verification.py @@ -360,6 +360,7 @@ def test_dflash_family_accept_uses_block_verification(any_greedy): corrected_logits=draft_logits, greedy_mask=greedy_mask, temperatures=torch.ones(batch, device=DEVICE), + greedy_draft=False, ), sampling_info=SimpleNamespace( is_all_greedy=False, diff --git a/test/registered/spec/dspark/test_dspark_draft_graph.py b/test/registered/spec/dspark/test_dspark_draft_graph.py index d46146ecac..2024dce253 100644 --- a/test/registered/spec/dspark/test_dspark_draft_graph.py +++ b/test/registered/spec/dspark/test_dspark_draft_graph.py @@ -1,8 +1,12 @@ import unittest from types import SimpleNamespace +from unittest import mock import torch +from sglang.kernels.ops.speculative.dspark.dspark_draft_model import ( + MarkovGreedyStep, +) from sglang.srt.models.dspark import VanillaMarkov from sglang.srt.speculative.dspark_components.dspark_draft import ( DraftNoise, @@ -69,6 +73,52 @@ class TestDraftMarkovBlockGraph(CustomTestCase): self.assertFalse(torch.equal(tokens_out[1::2], eager_tokens[1::2])) self.assertTrue(torch.equal(tokens_out[::2], eager_tokens[::2])) + def test_one_greedy_row_matches_eager_and_fp32_reference(self): + # The fused one-row kernel runs; the graph replays eager; both match FP32. + torch.manual_seed(2) + head = VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK).to(DEVICE) + head = head.to(torch.bfloat16) + base = torch.randn(1, GAMMA, VOCAB, device=DEVICE).to(torch.bfloat16) + anchor = torch.randint(0, VOCAB, (1,), device=DEVICE) + noise = DraftNoise( + key=3, offset=torch.zeros(1, dtype=torch.int64, device=DEVICE) + ) + tokens_out = torch.empty(1, GAMMA, dtype=torch.int64, device=DEVICE) + + def block(): + tokens, logits = draft_markov_block( + markov_head=head, + base_logits=base, + anchor_tokens=anchor, + draft_hidden=None, + step_temperatures=torch.zeros(1, device=DEVICE), + noise=noise, + tp_sync=SimpleNamespace(sync=lambda site, value: value), + greedy_draft=True, + ) + assert logits is None + tokens_out.copy_(tokens) + + with mock.patch.object( + MarkovGreedyStep, "execute", wraps=MarkovGreedyStep.execute + ) as fused: + block() + self.assertEqual(fused.call_count, GAMMA) + eager_tokens = tokens_out.clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + block() + tokens_out.zero_() + graph.replay() + torch.cuda.synchronize() + self.assertTrue(torch.equal(tokens_out, eager_tokens)) + + prev = anchor + for step in range(GAMMA): + bias = head.markov_w1(prev).float() @ head.markov_w2.weight.float().T + prev = (base[:, step].float() + bias).argmax(-1) + self.assertEqual(eager_tokens[0, step].item(), prev.item()) + def test_captured_sampler_matches_eager_block(self): # The real tail (staging, static buffers, [:bs] views) against eager. torch.manual_seed(1) @@ -92,6 +142,7 @@ class TestDraftMarkovBlockGraph(CustomTestCase): device=DEVICE, noise=noise, tp_sync=tp_sync, + greedy_draft=False, ) hidden = torch.randn(BS * GAMMA, 64, device=DEVICE).to(torch.bfloat16) input_ids = torch.randint(0, VOCAB, (BS * GAMMA,), device=DEVICE) @@ -121,6 +172,7 @@ class TestDraftMarkovBlockGraph(CustomTestCase): noise=eager_noise, device=DEVICE, tp_sync=tp_sync, + greedy_draft=False, ) self.assertTrue( torch.equal(sampler.out[: BS * GAMMA].view(BS, GAMMA), eager.draft_tokens) diff --git a/test/registered/unit/spec/test_dspark_draft_block.py b/test/registered/unit/spec/test_dspark_draft_block.py index 5efffe3f95..d1b8abac20 100644 --- a/test/registered/unit/spec/test_dspark_draft_block.py +++ b/test/registered/unit/spec/test_dspark_draft_block.py @@ -3,7 +3,12 @@ from types import SimpleNamespace import torch -from sglang.srt.models.dspark import RNNHead, VanillaMarkov +from sglang.srt.models.dspark import ( + GatedMarkovHead, + Nemotron35VanillaMarkov, + RNNHead, + VanillaMarkov, +) from sglang.srt.speculative.dspark_components.dspark_draft import ( DraftNoise, sample_draft_block, @@ -16,7 +21,7 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu") VOCAB, RANK, HIDDEN, BS, GAMMA = 211, 8, 16, 4, 5 -def _draft(head, sampling_info, offset): +def _draft(head, sampling_info, offset, greedy_draft=False): torch.manual_seed(0) base = torch.randn(BS, GAMMA, VOCAB).to(torch.bfloat16) hidden = torch.randn(BS, GAMMA, HIDDEN).to(torch.bfloat16) @@ -30,6 +35,7 @@ def _draft(head, sampling_info, offset): noise=DraftNoise(key=3, offset=offset), device="cpu", tp_sync=SimpleNamespace(sync=lambda site, tokens: tokens), + greedy_draft=greedy_draft, ) return block, base, hidden, anchor @@ -78,6 +84,45 @@ class TestSampleDraftBlock(CustomTestCase): self.assertEqual(block.greedy_mask.tolist(), greedy.tolist()) self.assertEqual(offset.item(), 1) + def test_greedy_draft_takes_every_argmax_and_keeps_no_logits(self): + head = VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK).to(torch.bfloat16) + # Sampling requests too: a greedy draft ignores their temperatures. + sampling_info = SimpleNamespace( + is_all_greedy=False, + temperatures=torch.tensor([[0.7], [1.0], [1.0], [1.3]]), + top_ks=torch.tensor([50, 50, 1, 50]), + ) + offset = torch.zeros(1, dtype=torch.int64) + block, base, hidden, anchor = _draft( + head, sampling_info, offset, greedy_draft=True + ) + logits = _step_logits_along(head, base, hidden, anchor, block.draft_tokens) + self.assertTrue(torch.equal(block.draft_tokens, logits.float().argmax(-1))) + self.assertIsNone(block.corrected_logits) + self.assertTrue(block.greedy_draft) + + +class TestFusedGreedyEligibility(CustomTestCase): + def test_only_the_plain_w2_gemv_head_fuses(self): + # A head that changes the bias must keep the shared op. + heads = ( + (VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK), True), + ( + Nemotron35VanillaMarkov( + vocab_size=VOCAB, markov_rank=RANK, quant_config=None, prefix="" + ), + False, + ), + ( + GatedMarkovHead(vocab_size=VOCAB, markov_rank=RANK, hidden_size=HIDDEN), + False, + ), + (RNNHead(vocab_size=VOCAB, markov_rank=RANK, hidden_size=HIDDEN), False), + ) + for head, fuses in heads: + with self.subTest(head=type(head).__name__): + self.assertIs(head.greedy_w2_weight() is not None, fuses) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/spec/test_dspark_greedy_draft_accept.py b/test/registered/unit/spec/test_dspark_greedy_draft_accept.py new file mode 100644 index 0000000000..c8708b112e --- /dev/null +++ b/test/registered/unit/spec/test_dspark_greedy_draft_accept.py @@ -0,0 +1,55 @@ +import unittest +from types import SimpleNamespace +from unittest import mock + +import torch + +from sglang.srt.speculative.dspark_components import dspark_verify +from sglang.srt.speculative.dspark_components.dspark_draft import DraftBlockResult +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +BS, GAMMA, VOCAB = 3, 4, 50 + + +class TestGreedyDraftAccept(CustomTestCase): + def test_sampling_accept_reads_a_one_hot_draft(self): + # The sampling accept gets a one-hot draft at the target vocab width. + torch.manual_seed(0) + draft_tokens = torch.randint(0, VOCAB, (BS, GAMMA)) + block = DraftBlockResult( + draft_tokens=draft_tokens, + corrected_logits=None, + greedy_mask=torch.zeros(BS, dtype=torch.bool), + temperatures=torch.ones(BS), + greedy_draft=True, + ) + seen = {} + + def fake_accept(**kwargs): + seen["draft_probs"] = kwargs["draft_probs"] + zeros = torch.zeros(BS, dtype=torch.int64) + return zeros, zeros, zeros + + with mock.patch.object( + dspark_verify.AcceptSampling, "execute", side_effect=fake_accept + ): + dspark_verify.accept_draft_tokens( + candidates=torch.zeros(BS, GAMMA + 1, dtype=torch.int64), + target_logits=torch.randn(BS * (GAMMA + 1), VOCAB), + draft_block=block, + sampling_info=SimpleNamespace(is_all_greedy=False, is_any_greedy=False), + draft_input=None, + gamma=GAMMA, + verify_num_draft_tokens=GAMMA + 1, + ) + probs = seen["draft_probs"] + self.assertEqual(probs.shape, (BS, GAMMA, VOCAB)) + expected = torch.nn.functional.one_hot(draft_tokens, VOCAB).float() + self.assertTrue(torch.equal(probs, expected)) + + +if __name__ == "__main__": + unittest.main()