diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index c28d839e56..410c9bf043 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -542,7 +542,6 @@ class Envs: SGLANG_DSPARK_ENABLE_SPS_RECORD = EnvBool(False) SGLANG_DSPARK_FAST_KERNEL = EnvBool(True) SGLANG_DSPARK_FP32_LM_HEAD = EnvBool(False) - SGLANG_DSPARK_FAST_SAMPLING = EnvBool(True) SGLANG_DSPARK_FOLDED_SAMPLING = EnvInt(DsparkFoldedSampling.AUTO) SGLANG_DSPARK_FOLDED_PROPOSAL = EnvBool(True) SGLANG_DSPARK_STACKED_CTX_KV = EnvBool(True) @@ -1988,6 +1987,9 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = { note="Strategy-based prefill context parallelism is now the only generic implementation." ), "SGLANG_TRACE_QWEN35_FINAL_NORM": _DeprecatedEnv(), + "SGLANG_DSPARK_FAST_SAMPLING": _DeprecatedEnv( + note="DSpark draft sampling always uses Gumbel-max." + ), "SGLANG_SKIP_RUST_TESTS": _DeprecatedEnv( note="The Rust tests run in pr-test-rust-workspace.yml now." ), diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index 6220e59768..74feb8aab8 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -51,6 +51,7 @@ from sglang.srt.models.deepseek_v4 import ( from sglang.srt.models.dspark import ( BaseDSparkMarkovHead, DSparkConfidenceHead, + VocabShard, gather_and_crop_vocab, project_through_lm_head, ) @@ -473,6 +474,9 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead): num_embeddings_per_partition=per_partition, num_embeddings_padded=num_padded, ) + self.vocab_shard = VocabShard( + start=self._tp_shard.org_vocab_start, gather=self._vocab_gather + ) def get_prev_embeddings(self, token_ids: torch.Tensor) -> torch.Tensor: return self.markov_w1(token_ids.long()) @@ -514,6 +518,11 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead): full = self._vocab_gather(step_local) return full[..., : self.vocab_size] + def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype: + # The bias is FP32 or upcast into an FP32 add, sharded or not. + del base_dtype + return torch.float32 + @property def supports_sharded_greedy(self) -> bool: return ( diff --git a/python/sglang/srt/models/dspark.py b/python/sglang/srt/models/dspark.py index a4734c3a53..e6b4422901 100644 --- a/python/sglang/srt/models/dspark.py +++ b/python/sglang/srt/models/dspark.py @@ -2,8 +2,9 @@ from __future__ import annotations import logging from abc import ABC, abstractmethod -from typing import Callable, Iterable, Optional, Tuple +from typing import TYPE_CHECKING, Callable, Iterable, Optional, Tuple +import msgspec import torch import torch.nn.functional as F from torch import nn @@ -28,6 +29,9 @@ from sglang.srt.speculative.ragged_verify import ( read_ragged_verify_mode, ) +if TYPE_CHECKING: + from sglang.srt.distributed.device_communicators.vocab_gather import VocabGather + logger = logging.getLogger(__name__) StepSampler = Callable[[torch.Tensor, int], torch.Tensor] @@ -85,10 +89,21 @@ def run_markov_block( ) +class VocabShard(msgspec.Struct, frozen=True): + """Where a rank's vocab slice starts, and the gather over its group.""" + + start: int + gather: VocabGather + + class BaseDSparkMarkovHead(nn.Module, ABC): """Corrects a draft block step by step: each step's base logits get a bias computed from the token drafted at the step before.""" + vocab_size: int + # The rank's slice of a TP-sharded vocab; None when the vocab is whole. + vocab_shard: Optional[VocabShard] = None + def init_state( self, hidden_states: Optional[torch.Tensor] ) -> Optional[torch.Tensor]: @@ -110,6 +125,10 @@ class BaseDSparkMarkovHead(nn.Module, ABC): ) -> torch.Tensor: return base_logits + bias + @abstractmethod + def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype: + """The dtype of ``base_logits + bias``.""" + @property def supports_sharded_greedy(self) -> bool: return False @@ -151,6 +170,10 @@ class VanillaMarkov(BaseDSparkMarkovHead): def project_bias(self, latent_states: torch.Tensor) -> torch.Tensor: return self.markov_w2(latent_states) + def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype: + # The bias comes out in the embedding's dtype. + return torch.promote_types(base_dtype, self.markov_w1.weight.dtype) + def step_bias( self, prev_tokens: torch.Tensor, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft.py b/python/sglang/srt/speculative/dspark_components/dspark_draft.py index 290d202e15..ea29e0fc30 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft.py @@ -1,14 +1,12 @@ from __future__ import annotations import logging -from typing import Optional +from typing import TYPE_CHECKING, Optional import msgspec import torch -from sglang.kernels.ops.speculative.dspark.dspark_draft_model import ( - SampleStepTokens, -) +from sglang.kernels.ops.speculative.dspark.markov_sample import MarkovSampleStep from sglang.srt.environ import envs from sglang.srt.lora.layers import unwrap_lora_layer from sglang.srt.managers.schedule_batch import ScheduleBatch @@ -30,24 +28,14 @@ from sglang.srt.speculative.spec_utils import draft_tp_context from sglang.srt.utils.common import is_pin_memory_available from sglang.srt.utils.invariants import Bucket, Invariant, NotNaN, expect +if TYPE_CHECKING: + from sglang.srt.models.dspark import BaseDSparkMarkovHead + logger = logging.getLogger(__name__) -def _one_hot_token0(probs: torch.Tensor) -> torch.Tensor: - degenerate = torch.isnan(probs[:, :1]) - one_hot = torch.zeros_like(probs) - one_hot[:, 0] = 1.0 - return torch.where(degenerate, one_hot, probs) - - -# Draft step logits: NaN is a bug, but -inf is legitimate (masking). The data -# layer lives downstream (probs one-hot below, or the fast kernel's clamp). +# Draft step logits: NaN is a bug, but -inf is legitimate (masking). _DRAFT_STEP_LOGITS = Invariant("dspark.draft.step_logits", Bucket.GUARD, NotNaN()) -# Draft sampling probs: SOFTEN (tolerate + count), matching the original -# unconditional clamp; an all-NaN row would otherwise make multinomial raise. -_DRAFT_PROBS = Invariant( - "dspark.draft.probs", Bucket.SOFTEN, NotNaN(), recover=_one_hot_token0 -) def _make_num_token_non_padded( @@ -125,74 +113,148 @@ def resolve_greedy_mask( return (sampling_info.top_ks <= 1).view(-1) +# XORed into the server seed so the draft noise differs from other samplers'. +_DRAFT_NOISE_DOMAIN = 0x2D2A9C317E44B1F3 + + +def draft_noise_key(random_seed: int) -> int: + return (int(random_seed) ^ _DRAFT_NOISE_DOMAIN) & 0x7FFFFFFFFFFFFFFF + + +class DraftNoise(msgspec.Struct, frozen=True): + """Philox key of the draft noise, and a GPU counter each block advances.""" + + key: int + offset: torch.Tensor + + +def step_logits_width(markov_head: BaseDSparkMarkovHead, base_width: int) -> int: + # Kept step logits are gathered whole over a sharded vocab. + return base_width if markov_head.vocab_shard is None else markov_head.vocab_size + + +def _draft_step( + markov_head: BaseDSparkMarkovHead, + prev_tokens: torch.Tensor, + base_logits: torch.Tensor, + *, + hidden_states: Optional[torch.Tensor], + state: Optional[torch.Tensor], + temperatures: torch.Tensor, + noise: DraftNoise, + step: int, + logits_out: Optional[torch.Tensor], +) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + bias, state = markov_head.step_bias( + prev_tokens, hidden_states=hidden_states, state=state + ) + shard = markov_head.vocab_shard + sharded = ( + {} + if shard is None + else dict( + vocab_start=shard.start, + gather=shard.gather.gather_stacked, + gather_logits=shard.gather, + ) + ) + tokens = MarkovSampleStep.execute( + base_logits=base_logits, + bias=bias, + temperatures=temperatures, + key=noise.key, + offset=noise.offset, + step=step, + step_dtype=markov_head.step_logits_dtype(base_logits.dtype), + logits_out=logits_out, + # Rows sample at their temperatures only when the step logits are kept. + has_sampled_rows=None if logits_out is not None else False, + **sharded, + ) + return tokens, state + + +def draft_markov_block( + *, + markov_head, + base_logits: torch.Tensor, + anchor_tokens: torch.Tensor, + draft_hidden: Optional[torch.Tensor], + step_temperatures: torch.Tensor, + noise: DraftNoise, + tp_sync: SpecTpSync, + site: SpecTpSyncSite, +) -> tuple[torch.Tensor, torch.Tensor]: + """Returns the drafted tokens and the [bs, gamma, vocab] step logits.""" + bs, gamma = base_logits.shape[:2] + logits_out = torch.empty( + (bs, gamma, step_logits_width(markov_head, base_logits.shape[-1])), + dtype=markov_head.step_logits_dtype(base_logits.dtype), + device=base_logits.device, + ) + state = markov_head.init_state(draft_hidden) + prev_tokens = anchor_tokens.long() + draft_tokens = [] + for step in range(gamma): + prev_tokens, state = _draft_step( + markov_head, + prev_tokens, + base_logits[:, step], + hidden_states=None if draft_hidden is None else draft_hidden[:, step], + state=state, + temperatures=step_temperatures, + noise=noise, + step=step, + logits_out=logits_out[:, step], + ) + prev_tokens = tp_sync.sync(site, prev_tokens) + draft_tokens.append(prev_tokens) + noise.offset.add_(1) + return torch.stack(draft_tokens, dim=1), logits_out + + +def resolve_draft_temperatures( + *, bs: int, sampling_info, device: torch.device +) -> torch.Tensor: + if sampling_info is None: + return torch.ones(bs, dtype=torch.float32, device=device) + return sampling_info.temperatures.view(-1).to(torch.float32).clamp_min(1e-5) + + def sample_draft_block( *, base_logits: torch.Tensor, anchor_tokens: torch.Tensor, - draft_hidden: torch.Tensor, + draft_hidden: Optional[torch.Tensor], sampling_info, markov_head, + noise: DraftNoise, device: torch.device, tp_sync: SpecTpSync, ) -> DraftBlockResult: bs = base_logits.shape[0] greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device) any_sampling = sampling_info is not None and not sampling_info.is_all_greedy - fast_sampling = envs.SGLANG_DSPARK_FAST_SAMPLING.get() - - if sampling_info is None: - temperatures = torch.ones(bs, dtype=torch.float32, device=device) - else: - temperatures = ( - sampling_info.temperatures.view(-1).to(torch.float32).clamp_min(1e-5) - ) - - if not any_sampling: - - def sampler(step_logits: torch.Tensor, step_idx: int) -> torch.Tensor: - expect(_DRAFT_STEP_LOGITS, step_logits, msg=f"step {step_idx}") - return tp_sync.sync( - SpecTpSyncSite.DSPARK_DRAFT_GREEDY, torch.argmax(step_logits, dim=-1) - ) - - else: - - def sampler(step_logits: torch.Tensor, step_idx: int) -> torch.Tensor: - expect(_DRAFT_STEP_LOGITS, step_logits, msg=f"step {step_idx}") - if fast_sampling: - exp_noise = torch.empty( - step_logits.shape, dtype=torch.float32, device=step_logits.device - ).exponential_(1) - return tp_sync.sync( - SpecTpSyncSite.DSPARK_DRAFT_SAMPLE, - SampleStepTokens.execute( - step_logits=step_logits, - temperatures=temperatures, - greedy_mask=greedy_mask, - exp_noise=exp_noise, - ), - ) - else: - probs = torch.softmax( - step_logits.float() / temperatures[:, None], dim=-1 - ) - probs = expect(_DRAFT_PROBS, probs) - argmax_tokens = torch.argmax(step_logits, dim=-1) - sampled_tokens = torch.multinomial(probs, num_samples=1).squeeze(-1) - return tp_sync.sync( - SpecTpSyncSite.DSPARK_DRAFT_MULTINOMIAL, - torch.where(greedy_mask, argmax_tokens, sampled_tokens), - ) - - draft_tokens, corrected_logits = markov_head.sample_block( - base_logits, - first_prev_tokens=anchor_tokens, - hidden_states=draft_hidden, - sampler=sampler, + temperatures = resolve_draft_temperatures( + bs=bs, sampling_info=sampling_info, device=device + ) + draft_tokens, step_logits = draft_markov_block( + markov_head=markov_head, + base_logits=base_logits, + anchor_tokens=anchor_tokens, + draft_hidden=draft_hidden, + step_temperatures=temperatures.masked_fill(greedy_mask, 0.0), + noise=noise, + tp_sync=tp_sync, + site=( + SpecTpSyncSite.DSPARK_DRAFT_SAMPLE + if any_sampling + else SpecTpSyncSite.DSPARK_DRAFT_GREEDY + ), ) return DraftBlockResult( draft_tokens=draft_tokens, - corrected_logits=corrected_logits, + corrected_logits=expect(_DRAFT_STEP_LOGITS, step_logits), greedy_mask=greedy_mask, temperatures=temperatures, ) @@ -208,6 +270,7 @@ class DraftBlockProposer: mask_token_id: int, draft_block_spec_info, tp_sync: SpecTpSync, + random_seed: int, dp_moe_sync: bool = False, ) -> None: self.draft_model = draft_model @@ -218,6 +281,12 @@ class DraftBlockProposer: self._mask_token_id = mask_token_id self._draft_block_spec_info = draft_block_spec_info self._tp_sync = tp_sync + self._noise = DraftNoise( + key=draft_noise_key(random_seed), + offset=torch.zeros( + 1, dtype=torch.int64, device=self.draft_model_runner.device + ), + ) self._draft_sampler = None self._dp_moe_sync = dp_moe_sync # Persistent (bs, gamma) mask-token buffer: only column 0 (the bonus @@ -322,6 +391,7 @@ class DraftBlockProposer: draft_hidden=fwd.draft_hidden_3d, sampling_info=sampling_info, markov_head=self.draft_model.markov_head, + noise=self._noise, device=device, tp_sync=self._tp_sync, ) diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index 49ca5f0029..134ff74ba3 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -323,6 +323,7 @@ class DSparkWorkerV2(BaseSpecWorker): mask_token_id=self._mask_token_id, draft_block_spec_info=self._draft_block_spec_info, tp_sync=self._tp_sync, + random_seed=target_worker.random_seed, dp_moe_sync=self._draft_is_moe and get_parallel().attn_dp_enabled, ) self._verify_epilogue = None diff --git a/test/registered/unit/spec/test_dspark_draft_block.py b/test/registered/unit/spec/test_dspark_draft_block.py new file mode 100644 index 0000000000..5efffe3f95 --- /dev/null +++ b/test/registered/unit/spec/test_dspark_draft_block.py @@ -0,0 +1,83 @@ +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.models.dspark import RNNHead, VanillaMarkov +from sglang.srt.speculative.dspark_components.dspark_draft import ( + DraftNoise, + sample_draft_block, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +VOCAB, RANK, HIDDEN, BS, GAMMA = 211, 8, 16, 4, 5 + + +def _draft(head, sampling_info, offset): + torch.manual_seed(0) + base = torch.randn(BS, GAMMA, VOCAB).to(torch.bfloat16) + hidden = torch.randn(BS, GAMMA, HIDDEN).to(torch.bfloat16) + anchor = torch.randint(0, VOCAB, (BS,)) + block = sample_draft_block( + base_logits=base, + anchor_tokens=anchor, + draft_hidden=hidden, + sampling_info=sampling_info, + markov_head=head, + noise=DraftNoise(key=3, offset=offset), + device="cpu", + tp_sync=SimpleNamespace(sync=lambda site, tokens: tokens), + ) + return block, base, hidden, anchor + + +def _step_logits_along(head, base, hidden, anchor, tokens): + state = head.init_state(hidden) + prev = anchor + rows = [] + for step in range(GAMMA): + bias, state = head.step_bias(prev, hidden_states=hidden[:, step], state=state) + rows.append(base[:, step] + bias) + prev = tokens[:, step] + return torch.stack(rows, dim=1) + + +class TestSampleDraftBlock(CustomTestCase): + def test_greedy_rows_take_the_argmax_and_keep_their_logits(self): + for head in ( + VanillaMarkov(vocab_size=VOCAB, markov_rank=RANK), + RNNHead(vocab_size=VOCAB, markov_rank=RANK, hidden_size=HIDDEN), + ): + head = head.to(torch.bfloat16) + # top_k <= 1 marks a greedy request; rows 0 and 2 are greedy. + sampling_info = SimpleNamespace( + is_all_greedy=False, + temperatures=torch.tensor([[0.7], [1.0], [1.0], [1.3]]), + top_ks=torch.tensor([1, 50, 1, 50]), + ) + offset = torch.zeros(1, dtype=torch.int64) + block, base, hidden, anchor = _draft(head, sampling_info, offset) + logits = _step_logits_along(head, base, hidden, anchor, block.draft_tokens) + self.assertEqual(block.corrected_logits.dtype, logits.dtype) + self.assertTrue(torch.equal(block.corrected_logits, logits)) + greedy = torch.tensor([True, False, True, False]) + self.assertTrue( + torch.equal( + block.draft_tokens[greedy], logits[greedy].float().argmax(-1) + ) + ) + # The sampled rows drew noise: they leave the argmax somewhere. + self.assertFalse( + torch.equal( + block.draft_tokens[~greedy], logits[~greedy].float().argmax(-1) + ) + ) + self.assertEqual(block.greedy_mask.tolist(), greedy.tolist()) + self.assertEqual(offset.item(), 1) + + +if __name__ == "__main__": + unittest.main()