diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index 318c52d9ae..62b5776658 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -65,16 +65,11 @@ from sglang.srt.speculative.ragged_verify import ( read_ragged_verify_mode, ) from sglang.srt.utils import add_prefix, is_npu -from sglang.srt.utils.invariants import Bucket, InClosedRange, Invariant, expect logger = logging.getLogger(__name__) _PAD_NUM_HEADS = 64 -# DSpark confidence is a per-token score that must stay in [0, 1]. -_CONFIDENCE = Invariant( - "dspark.model.confidence", Bucket.GUARD, InClosedRange(0.0, 1.0) -) _is_npu = is_npu() @@ -1002,25 +997,15 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): self, *, anchor_tokens: torch.Tensor, - sampled_tokens: torch.Tensor, - x_post_hc: torch.Tensor, - ) -> Optional[torch.Tensor]: - confidence_head = self.confidence_head - if confidence_head is None: - return None - bs = int(anchor_tokens.shape[0]) - x_post_hc = x_post_hc.view(bs, self.gamma, -1) - if confidence_head.with_markov: - prev_seq = torch.cat( - [anchor_tokens.view(-1, 1), sampled_tokens[:, : self.gamma - 1]], dim=1 - ) - markov_embed_stack = self.markov_head.get_prev_embeddings(prev_seq) - else: - markov_embed_stack = None - confidence_raw = confidence_head(x_post_hc, markov_embed_stack) - confidence = confidence_head.apply_sts(confidence_raw) - expect(_CONFIDENCE, confidence) - return confidence + draft_tokens: torch.Tensor, + confidence_features: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + return self.confidence_head.predict( + confidence_features, + anchor_tokens=anchor_tokens, + draft_tokens=draft_tokens, + markov_head=self.markov_head, + ) def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> None: params_dict = dict(self.named_parameters()) diff --git a/python/sglang/srt/models/dspark.py b/python/sglang/srt/models/dspark.py index 000c6bed9e..91f17f9bed 100644 --- a/python/sglang/srt/models/dspark.py +++ b/python/sglang/srt/models/dspark.py @@ -25,12 +25,18 @@ from sglang.srt.speculative.ragged_verify import ( RaggedVerifyMode, read_ragged_verify_mode, ) +from sglang.srt.utils.invariants import Bucket, InClosedRange, Invariant, expect if TYPE_CHECKING: from sglang.srt.distributed.device_communicators.vocab_gather import VocabGather logger = logging.getLogger(__name__) +# DSpark confidence is a per-token score that must stay in [0, 1]. +_CONFIDENCE = Invariant( + "dspark.model.confidence", Bucket.GUARD, InClosedRange(0.0, 1.0) +) + def gather_and_crop_vocab( local_logits: torch.Tensor, lm_head: nn.Module @@ -299,7 +305,6 @@ class DSparkConfidenceHead(nn.Module): self.register_buffer( "sts_temperatures", torch.ones((), dtype=torch.float32), persistent=False ) - self._last_confidence_raw: Optional[torch.Tensor] = None def forward( self, @@ -321,9 +326,30 @@ class DSparkConfidenceHead(nn.Module): return self.proj(features).squeeze(-1) def apply_sts(self, confidence_raw: torch.Tensor) -> torch.Tensor: - self._last_confidence_raw = confidence_raw return torch.sigmoid(confidence_raw.float() / self.sts_temperatures) + def predict( + self, + features: torch.Tensor, + *, + anchor_tokens: torch.Tensor, + draft_tokens: torch.Tensor, + markov_head: BaseDSparkMarkovHead, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """The [bs, gamma] confidence of a drafted block and its raw logit.""" + bs, gamma = draft_tokens.shape + if self.with_markov: + prev_tokens = torch.cat( + [anchor_tokens.view(-1, 1), draft_tokens[:, : gamma - 1]], dim=1 + ) + markov_embed_stack = markov_head.get_prev_embeddings(prev_tokens) + else: + markov_embed_stack = None + confidence_raw = self(features.view(bs, gamma, -1), markov_embed_stack) + confidence = self.apply_sts(confidence_raw) + expect(_CONFIDENCE, confidence) + return confidence, confidence_raw + def build_confidence_head(config) -> Optional[nn.Module]: if read_ragged_verify_mode() is RaggedVerifyMode.STATIC: @@ -399,8 +425,9 @@ class DSparkDraftMixin: def compute_base_logits( self, hidden: torch.Tensor - ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: - """Project the draft's raw final hidden through the target lm_head. + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Project the draft's raw final hidden through the target lm_head; + also returns the features the confidence head reads. muP targets (Inkling) train the draft against a FOLDED head (weights pre-divided by logits_mup_width_multiplier) while serving attaches the @@ -414,11 +441,26 @@ class DSparkDraftMixin: "DSpark dense draft requires the target lm_head " "(call attach_shared_modules first)." ) + confidence_features = hidden if self.logits_mup_width_multiplier: hidden = hidden / self.logits_mup_width_multiplier local_logits = project_through_lm_head(hidden, self.lm_head) base_logits = gather_and_crop_vocab(local_logits, self.lm_head) - return base_logits, None + return base_logits, confidence_features + + def compute_confidence( + self, + *, + anchor_tokens: torch.Tensor, + draft_tokens: torch.Tensor, + confidence_features: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + return self.confidence_head.predict( + confidence_features, + anchor_tokens=anchor_tokens, + draft_tokens=draft_tokens, + markov_head=self.markov_head, + ) def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): markov_weights = [] diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft.py b/python/sglang/srt/speculative/dspark_components/dspark_draft.py index eebd6d89bc..a02bb30e10 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft.py @@ -1,7 +1,8 @@ from __future__ import annotations +import functools import logging -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, NamedTuple, Optional import msgspec import torch @@ -72,9 +73,9 @@ class DraftForwardResult(msgspec.Struct, frozen=True): class DraftProposal(msgspec.Struct, frozen=True): draft_block_ids: torch.Tensor draft_block: DraftBlockResult - draft_hidden: Optional[torch.Tensor] + # [bs, gamma]; None when the draft has no confidence head. confidence: Optional[torch.Tensor] = None - confidence_tap: Optional[torch.Tensor] = None + confidence_raw: Optional[torch.Tensor] = None folded: bool = False @@ -118,6 +119,51 @@ def resolve_greedy_mask( return (sampling_info.top_ks <= 1).view(-1) +class DraftParams(NamedTuple): + temperatures: torch.Tensor + greedy_mask: torch.Tensor + # The temperature each step samples at: 0 on argmax rows. + step_temperatures: torch.Tensor + + +@functools.cache +def _zero(device: torch.device) -> torch.Tensor: + return torch.zeros((), dtype=torch.float32, device=device) + + +def resolve_draft_params( + *, + sampling_info, + bs: int, + device: Optional[torch.device] = None, + out: Optional[DraftParams] = None, +) -> DraftParams: + """Written into the ``[:bs]`` rows of ``out`` when given (the graph's + static buffers), else into fresh tensors on ``device``.""" + if out is not None: + device = out.temperatures.device + else: + out = DraftParams( + temperatures=torch.empty(bs, dtype=torch.float32, device=device), + greedy_mask=torch.empty(bs, dtype=torch.bool, device=device), + step_temperatures=torch.empty(bs, dtype=torch.float32, device=device), + ) + temperatures, greedy_mask, step_temperatures = (buf[:bs] for buf in out) + if sampling_info is None: + temperatures.fill_(1.0) + greedy_mask.fill_(True) + step_temperatures.zero_() + else: + torch.clamp( + sampling_info.temperatures.view(-1)[:bs].to(torch.float32), + min=1e-5, + out=temperatures, + ) + torch.le(sampling_info.top_ks.view(-1)[:bs], 1, out=greedy_mask) + torch.where(greedy_mask, _zero(device), temperatures, out=step_temperatures) + return DraftParams(temperatures, greedy_mask, step_temperatures) + + # XORed into the server seed so the draft noise differs from other samplers'. _DRAFT_NOISE_DOMAIN = 0x2D2A9C317E44B1F3 @@ -234,14 +280,6 @@ def draft_markov_block( return torch.stack(draft_tokens, dim=1), logits_out -def resolve_draft_temperatures( - *, bs: int, sampling_info, device: torch.device -) -> torch.Tensor: - if sampling_info is None: - return torch.ones(bs, dtype=torch.float32, device=device) - return sampling_info.temperatures.view(-1).to(torch.float32).clamp_min(1e-5) - - def sample_draft_block( *, base_logits: torch.Tensor, @@ -254,17 +292,15 @@ def sample_draft_block( tp_sync: SpecTpSync, greedy_draft: bool, ) -> DraftBlockResult: - bs = base_logits.shape[0] - greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device) - temperatures = resolve_draft_temperatures( - bs=bs, sampling_info=sampling_info, device=device + params = resolve_draft_params( + sampling_info=sampling_info, bs=base_logits.shape[0], device=device ) draft_tokens, step_logits = draft_markov_block( markov_head=markov_head, base_logits=base_logits, anchor_tokens=anchor_tokens, draft_hidden=draft_hidden, - step_temperatures=temperatures.masked_fill(greedy_mask, 0.0), + step_temperatures=params.step_temperatures, noise=noise, tp_sync=tp_sync, greedy_draft=greedy_draft, @@ -274,8 +310,8 @@ def sample_draft_block( corrected_logits=( None if greedy_draft else expect(_DRAFT_STEP_LOGITS, step_logits) ), - greedy_mask=greedy_mask, - temperatures=temperatures, + greedy_mask=params.greedy_mask, + temperatures=params.temperatures, greedy_draft=greedy_draft, ) @@ -359,8 +395,7 @@ class DraftBlockProposer: ) draft_block_ids = fwd.draft_block_ids - folded_confidence = None - confidence_tap = None + confidence = confidence_raw = None folded = False if ( envs.SGLANG_DSPARK_FOLDED_PROPOSAL.get() @@ -373,15 +408,16 @@ class DraftBlockProposer: corrected_logits=( None if self._greedy_draft else draft_sampler.step_logits[:bs] ), - greedy_mask=draft_sampler.greedy_mask[:bs], - temperatures=draft_sampler.temperatures[:bs], + greedy_mask=draft_sampler.params.greedy_mask[:bs], + temperatures=draft_sampler.params.temperatures[:bs], greedy_draft=self._greedy_draft, ) if draft_sampler.confidence_out is not None: - folded_confidence = draft_sampler.confidence_out[:bs] + confidence = draft_sampler.confidence_out[:bs] + confidence_raw = draft_sampler.confidence_raw_out[:bs] else: with self._base_logits_context(): - base_logits, confidence_tap = self.draft_model.compute_base_logits( + base_logits, confidence_features = self.draft_model.compute_base_logits( fwd.raw_hidden ) base_logits = base_logits.view(bs, self.gamma, -1) @@ -396,6 +432,12 @@ class DraftBlockProposer: tp_sync=self._tp_sync, greedy_draft=self._greedy_draft, ) + if self.draft_model.confidence_head is not None: + confidence, confidence_raw = self.draft_model.compute_confidence( + anchor_tokens=draft_block_ids[:, 0], + draft_tokens=draft_block.draft_tokens, + confidence_features=confidence_features, + ) proposal_block_ids = ( draft_block_ids if self.sample_from_anchor @@ -404,9 +446,8 @@ class DraftBlockProposer: return DraftProposal( draft_block_ids=proposal_block_ids, draft_block=draft_block, - draft_hidden=fwd.draft_hidden_3d, - confidence=folded_confidence, - confidence_tap=confidence_tap, + confidence=confidence, + confidence_raw=confidence_raw, folded=folded, ) diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py index 64f1f9160c..3797bbcae4 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py @@ -7,7 +7,9 @@ import torch from sglang.srt.speculative.dspark_components.dspark_draft import ( DraftNoise, + DraftParams, draft_markov_block, + resolve_draft_params, select_draft_hidden_without_anchor, step_logits_width, ) @@ -29,7 +31,6 @@ class DsparkDraftSampler: noise: DraftNoise, tp_sync: SpecTpSync, greedy_draft: bool, - confidence_fn=None, out=None, ): self.model = model @@ -46,55 +47,47 @@ class DsparkDraftSampler: self.out = torch.empty( (self.max_bs * self.gamma,), dtype=torch.int64, device=device ) - self.confidence_fn = confidence_fn - self.confidence_out = ( - torch.empty((self.max_bs, self.gamma), dtype=torch.float32, device=device) - if confidence_fn is not None - else None - ) + confidence_head = model.confidence_head + if confidence_head is None: + self.confidence_out = self.confidence_raw_out = None + else: + self.confidence_out = torch.empty( + (self.max_bs, self.gamma), dtype=torch.float32, device=device + ) + self.confidence_raw_out = torch.empty( + (self.max_bs, self.gamma), + dtype=confidence_head.proj.weight.dtype, + device=device, + ) self.greedy_draft = greedy_draft self._noise = noise self._tp_sync = tp_sync - # Staged before each replay; step_temperatures is 0 on argmax rows. - self.temperatures = torch.ones( - (self.max_bs,), dtype=torch.float32, device=device + # Staged before each replay. + self.params = DraftParams( + temperatures=torch.ones((self.max_bs,), dtype=torch.float32, device=device), + greedy_mask=torch.ones((self.max_bs,), dtype=torch.bool, device=device), + step_temperatures=torch.zeros( + (self.max_bs,), dtype=torch.float32, device=device + ), ) - self.greedy_mask = torch.ones((self.max_bs,), dtype=torch.bool, device=device) - self.step_temperatures = torch.zeros( - (self.max_bs,), dtype=torch.float32, device=device - ) - self._zero = torch.zeros((), dtype=torch.float32, device=device) - # [max_bs, gamma, vocab] step logits for the sampling accept; a greedy - # draft keeps none. - self.step_logits = ( - None - if greedy_draft - else torch.empty( - (self.max_bs, self.gamma, _step_logits_width(model)), - dtype=_step_logits_dtype(model), - device=device, + # [max_bs, gamma, vocab] step logits for the sampling accept, sized on + # the first call; a greedy draft keeps none. + self.step_logits: Optional[torch.Tensor] = None + + def _step_logits(self, base_logits: torch.Tensor) -> torch.Tensor: + if self.step_logits is None: + # The runner's warmup calls the tail eagerly before capture. + assert not torch.cuda.is_current_stream_capturing() + self.step_logits = torch.empty( + (self.max_bs, self.gamma, _step_logits_width(self.model)), + dtype=self.markov_head.step_logits_dtype(base_logits.dtype), + device=base_logits.device, ) - ) + return self.step_logits def stage_sampling_params(self, *, bs: int, sampling_info) -> None: """Must run before the draft graph replay that reads these buffers.""" - if sampling_info is None: - self.temperatures[:bs].fill_(1.0) - self.greedy_mask[:bs].fill_(True) - self.step_temperatures[:bs].zero_() - return - torch.clamp( - sampling_info.temperatures.view(-1)[:bs].to(torch.float32), - min=1e-5, - out=self.temperatures[:bs], - ) - torch.le(sampling_info.top_ks.view(-1)[:bs], 1, out=self.greedy_mask[:bs]) - torch.where( - self.greedy_mask[:bs], - self._zero, - self.temperatures[:bs], - out=self.step_temperatures[:bs], - ) + resolve_draft_params(sampling_info=sampling_info, bs=bs, out=self.params) def __call__(self, hidden_states, input_ids): bs = hidden_states.shape[0] // self.query_token_num @@ -107,7 +100,7 @@ class DsparkDraftSampler: bs=bs, gamma=self.gamma, ) - base_logits, confidence_tap = self.model.compute_base_logits(model_hidden) + base_logits, confidence_features = self.model.compute_base_logits(model_hidden) base_logits = base_logits.view(bs, self.gamma, -1) anchor = input_ids.view(bs, self.query_token_num)[:, 0] @@ -116,21 +109,23 @@ class DsparkDraftSampler: base_logits=base_logits, anchor_tokens=anchor, draft_hidden=sample_hidden, - step_temperatures=self.step_temperatures[:bs], + step_temperatures=self.params.step_temperatures[:bs], noise=self._noise, tp_sync=self._tp_sync, - logits_out=None if self.greedy_draft else self.step_logits[:bs], + logits_out=None + if self.greedy_draft + else self._step_logits(base_logits)[:bs], greedy_draft=self.greedy_draft, ) self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1)) if self.confidence_out is not None: - confidence = self.confidence_fn( - draft_hidden=sample_hidden, + confidence, confidence_raw = self.model.compute_confidence( anchor_tokens=anchor, draft_tokens=draft_tokens, - confidence_tap=confidence_tap, + confidence_features=confidence_features, ) self.confidence_out[:bs].copy_(confidence) + self.confidence_raw_out[:bs].copy_(confidence_raw) def _step_logits_width(model) -> int: @@ -138,16 +133,6 @@ def _step_logits_width(model) -> int: return step_logits_width(model.markov_head, int(model.lm_head.org_vocab_size)) -def _step_logits_dtype(model) -> torch.dtype: - # A quantized head's packed weight has no logits dtype; it emits the draft's. - weight = model.lm_head.weight - if weight.is_floating_point(): - base_dtype = weight.dtype - else: - base_dtype = next(model.markov_head.parameters()).dtype - return model.markov_head.step_logits_dtype(base_dtype) - - def maybe_build_draft_sampler( *, draft_model, @@ -158,7 +143,6 @@ def maybe_build_draft_sampler( noise: DraftNoise, greedy_draft: bool, tp_sync: SpecTpSync, - confidence_fn=None, out=None, ) -> Optional[DsparkDraftSampler]: """Build the graph-folded draft sampler, or None (reason logged) when the @@ -186,6 +170,5 @@ def maybe_build_draft_sampler( noise=noise, tp_sync=tp_sync, greedy_draft=greedy_draft, - confidence_fn=confidence_fn, out=out, ) diff --git a/python/sglang/srt/speculative/dspark_components/dspark_observability.py b/python/sglang/srt/speculative/dspark_components/dspark_observability.py index 0a1c026f55..ed961b1315 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_observability.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_observability.py @@ -824,6 +824,7 @@ class DsparkStepObservers: target_logits: Optional[torch.Tensor], layout, confidence: Optional[torch.Tensor], + confidence_raw: Optional[torch.Tensor], prefix_lens: torch.Tensor, draft_tokens: torch.Tensor, draft_block, @@ -838,20 +839,20 @@ class DsparkStepObservers: dp_tier_num_tokens: Optional[int], ) -> None: planner = self._planner - if not proposal_folded: - self._maybe_record_sts_collect( - verify_ids_2d=verify_ids_2d, - target_logits=target_logits, - bs=bs, - ) - self._confidence_probe.maybe_observe( - carries_confidence=planner.carries_confidence, - is_compact_mode=planner.is_compact_mode, - confidence_raw=planner.last_confidence_raw, - verify_ids_2d=verify_ids_2d, - target_logits=target_logits, - bs=bs, - ) + self._maybe_record_sts_collect( + confidence_raw=confidence_raw, + verify_ids_2d=verify_ids_2d, + target_logits=target_logits, + bs=bs, + ) + self._confidence_probe.maybe_observe( + carries_confidence=planner.carries_confidence, + is_compact_mode=planner.is_compact_mode, + confidence_raw=confidence_raw, + verify_ids_2d=verify_ids_2d, + target_logits=target_logits, + bs=bs, + ) if self._block_accept_recorder is not None and not proposal_folded: self._block_accept_recorder.observe_verify_step( forward_ct=forward_ct, @@ -929,15 +930,13 @@ class DsparkStepObservers: def _maybe_record_sts_collect( self, *, + confidence_raw: Optional[torch.Tensor], verify_ids_2d: torch.Tensor, target_logits: Optional[torch.Tensor], bs: int, ) -> None: if not self._sts_collect_path: return - if not self._planner.carries_confidence: - return - confidence_raw = self._planner.last_confidence_raw if confidence_raw is None: return if self._sts_recorder is None: diff --git a/python/sglang/srt/speculative/dspark_components/dspark_planner.py b/python/sglang/srt/speculative/dspark_components/dspark_planner.py index 95e50081d8..f0793f3fc5 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_planner.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_planner.py @@ -45,7 +45,6 @@ from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite from sglang.srt.utils.common import require_mlp_tp_gather from sglang.srt.utils.invariants import ( Bucket, - InClosedRange, Invariant, IsTrue, expect, @@ -54,10 +53,6 @@ from sglang.srt.utils.invariants import ( logger = logging.getLogger(__name__) -# DSpark confidence is a per-token score that must stay in [0, 1]. -_CONFIDENCE = Invariant( - "dspark.planner.confidence", Bucket.GUARD, InClosedRange(0.0, 1.0) -) # Scheduled verify lengths must not exceed the per-step token budget. _VERIFY_LEN_BUDGET = Invariant("dspark.verify_len_budget", Bucket.GUARD, IsTrue()) @@ -214,12 +209,6 @@ class DSparkVerifyPlanner: def carries_confidence(self) -> bool: return self._confidence_head is not None - @property - def last_confidence_raw(self) -> Optional[torch.Tensor]: - if self._confidence_head is None: - return None - return self._confidence_head._last_confidence_raw - @property def schedules_verify_budget(self) -> bool: return self._budget_planner is not None @@ -252,37 +241,6 @@ class DSparkVerifyPlanner: self._ragged_verify_mode is RaggedVerifyMode.COMPACT and layout is not None ) - def compute_confidence_tensor( - self, - *, - draft_hidden: Optional[torch.Tensor], - anchor_tokens: torch.Tensor, - draft_tokens: torch.Tensor, - confidence_tap: Optional[torch.Tensor] = None, - ) -> Optional[torch.Tensor]: - if self._confidence_head is None: - return None - compute_confidence_hook = getattr(self.draft_model, "compute_confidence", None) - if compute_confidence_hook is not None: - assert confidence_tap is not None, ( - "dsv4 compute_confidence needs the compute_base_logits tap" - ) - with torch.inference_mode(): - return compute_confidence_hook( - anchor_tokens=anchor_tokens, - sampled_tokens=draft_tokens, - x_post_hc=confidence_tap, - ) - assert draft_hidden is not None - return compute_confidence( - draft_hidden=draft_hidden, - anchor_tokens=anchor_tokens, - draft_tokens=draft_tokens, - confidence_head=self._confidence_head, - markov_head=self.draft_model.markov_head, - gamma=self.gamma, - ) - def prepare_verify_budget( self, batch: ScheduleBatch, future_map: FutureMap ) -> None: @@ -877,44 +835,6 @@ def apply_logits_adjustments_strided( ) -def build_markov_embed_stack( - *, - anchor_tokens: torch.Tensor, - draft_tokens: torch.Tensor, - markov_head, - gamma: int, -) -> torch.Tensor: - prev_seq = torch.cat( - [anchor_tokens.view(-1, 1), draft_tokens[:, : gamma - 1]], dim=1 - ) - return markov_head.get_prev_embeddings(prev_seq) - - -def compute_confidence( - *, - draft_hidden: torch.Tensor, - anchor_tokens: torch.Tensor, - draft_tokens: torch.Tensor, - confidence_head, - markov_head, - gamma: int, -) -> torch.Tensor: - assert confidence_head is not None - if confidence_head.with_markov: - markov_embed_stack = build_markov_embed_stack( - anchor_tokens=anchor_tokens, - draft_tokens=draft_tokens, - markov_head=markov_head, - gamma=gamma, - ) - else: - markov_embed_stack = None - confidence_raw = confidence_head(draft_hidden, markov_embed_stack) - confidence = confidence_head.apply_sts(confidence_raw) - expect(_CONFIDENCE, confidence) - return confidence - - class DSparkScheduleConfig(msgspec.Struct): gamma: int min_verify_len: int = 1 diff --git a/python/sglang/srt/speculative/dspark_components/dspark_verify.py b/python/sglang/srt/speculative/dspark_components/dspark_verify.py index 0c11fca214..a6aaac3af4 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_verify.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_verify.py @@ -14,8 +14,6 @@ from sglang.kernels.ops.speculative.dspark.dspark_accept import ( FinalizeAcceptLens, SelectMixedAccept, SoftmaxTemp, - accept_greedy_triton, - finalize_accept_lens_triton, ) from sglang.kernels.ops.speculative.dspark.dspark_verify_window import ( BuildCommitInjectLayout, @@ -183,46 +181,24 @@ class TargetVerifyExecutor: fused_argmax=self._target_is_dsv41, block_verification=self._block_verification, ) - if simulate: - correct_len = self._simulated_correct_len.repeat(bs).to(correct_len.dtype) - if simulate_bonus_sampling_info is not None: - bonus = sample_simulated_bonus( - target_logits=target_logits, - correct_len=correct_len, - greedy_bonus=bonus, - sampling_info=simulate_bonus_sampling_info, - bs=bs, - verify_num_draft_tokens=self.verify_num_draft_tokens, - ) - - site = ( + sync_site = ( SpecTpSyncSite.DSPARK_ACCEPT_GREEDY if accept_sampling_info is None or accept_sampling_info.is_all_greedy else SpecTpSyncSite.DSPARK_ACCEPT_SAMPLE ) - self._tp_sync.sync(site, correct_len) - self._tp_sync.sync(site, bonus) - self._tp_sync.sync(site, cap_trim_lens) - - finalized = FinalizeAcceptLens.execute( + return build_accept_outs( correct_len=correct_len, + bonus=bonus, cap_trim_lens=cap_trim_lens, prefix_lens=prefix_lens, - ) - out_tokens = BuildOutTokens.execute( draft_tokens=draft_tokens, - correct_len=correct_len, - bonus=bonus, - verify_num_draft_tokens=self.verify_num_draft_tokens, gamma=self.gamma, - ) - return AcceptOuts( - correct_len=correct_len, - bonus=bonus, - cap_trim_lens=finalized.cap_trim_lens, - commit_lens=finalized.commit_lens, - new_seq_lens=finalized.new_seq_lens, - out_tokens=out_tokens, + verify_num_draft_tokens=self.verify_num_draft_tokens, + tp_sync=self._tp_sync, + sync_site=sync_site, + simulated_correct_len=self._simulated_correct_len, + target_logits=target_logits, + simulate_bonus_sampling_info=simulate_bonus_sampling_info, ) def begin_simulated_step(self) -> None: @@ -768,38 +744,27 @@ class DsparkVerifyEpilogue: self, *, candidates, logits, draft_tokens, seq_lens, cutoff_verify_lens=None ) -> torch.Tensor: bs = candidates.shape[0] - correct_len, bonus, cap_trim_lens = accept_greedy_triton( + correct_len, bonus, cap_trim_lens = AcceptGreedy.execute( candidates=candidates, target_logits=logits, verify_num_draft_tokens=self.stride, cutoff_verify_lens=cutoff_verify_lens, fused_argmax=self._fused_argmax, ) - if self.simulated_correct_len is not None: - # The accept runs as served; only its length is replaced. - correct_len = self.simulated_correct_len.repeat(bs).to(correct_len.dtype) - self._tp_sync.sync(SpecTpSyncSite.DSPARK_ACCEPT_GRAPH, correct_len) - self._tp_sync.sync(SpecTpSyncSite.DSPARK_ACCEPT_GRAPH, bonus) - self._tp_sync.sync(SpecTpSyncSite.DSPARK_ACCEPT_GRAPH, cap_trim_lens) - finalized = finalize_accept_lens_triton( - correct_len=correct_len, - cap_trim_lens=cap_trim_lens, - prefix_lens=seq_lens[:bs], - ) - out_tokens = BuildOutTokens.execute( - draft_tokens=draft_tokens, + accept = build_accept_outs( correct_len=correct_len, bonus=bonus, - verify_num_draft_tokens=self.stride, + cap_trim_lens=cap_trim_lens, + prefix_lens=seq_lens[:bs], + draft_tokens=draft_tokens, gamma=self.gamma, + verify_num_draft_tokens=self.stride, + tp_sync=self._tp_sync, + sync_site=SpecTpSyncSite.DSPARK_ACCEPT_GRAPH, + simulated_correct_len=self.simulated_correct_len, + out=self.read_accept(bs), ) - self.correct_len_buf[:bs].copy_(correct_len) - self.bonus_buf[:bs].copy_(bonus) - self.cap_trim_lens_buf[:bs].copy_(cap_trim_lens.to(torch.int32)) - self.commit_lens_buf[:bs].copy_(finalized.commit_lens) - self.new_seq_lens_buf[:bs].copy_(finalized.new_seq_lens) - self.out_tokens_buf[:bs].copy_(out_tokens.view(bs, self.stride)) - return finalized.commit_lens + return accept.commit_lens def _commit_inject( self, commit_lens, verify_lens, seq_lens, req_pool_indices, bs: int @@ -838,6 +803,72 @@ class DsparkVerifyEpilogue: ) +def build_accept_outs( + *, + correct_len: torch.Tensor, + bonus: torch.Tensor, + cap_trim_lens: torch.Tensor, + prefix_lens: torch.Tensor, + draft_tokens: torch.Tensor, + gamma: int, + verify_num_draft_tokens: int, + tp_sync: SpecTpSync, + sync_site: SpecTpSyncSite, + simulated_correct_len: Optional[torch.Tensor] = None, + target_logits: Optional[torch.Tensor] = None, + simulate_bonus_sampling_info=None, + out: Optional[AcceptOuts] = None, +) -> AcceptOuts: + """``out``: static buffers (the in-graph accept's) to copy the results into.""" + bs = correct_len.shape[0] + if simulated_correct_len is not None: + # The accept runs as served; only its length is replaced. + correct_len = simulated_correct_len.repeat(bs).to(correct_len.dtype) + if simulate_bonus_sampling_info is not None: + assert target_logits is not None + bonus = sample_simulated_bonus( + target_logits=target_logits, + correct_len=correct_len, + greedy_bonus=bonus, + sampling_info=simulate_bonus_sampling_info, + bs=bs, + verify_num_draft_tokens=verify_num_draft_tokens, + ) + tp_sync.sync(sync_site, correct_len) + tp_sync.sync(sync_site, bonus) + tp_sync.sync(sync_site, cap_trim_lens) + + finalized = FinalizeAcceptLens.execute( + correct_len=correct_len, + cap_trim_lens=cap_trim_lens, + prefix_lens=prefix_lens, + ) + out_tokens = BuildOutTokens.execute( + draft_tokens=draft_tokens, + correct_len=correct_len, + bonus=bonus, + verify_num_draft_tokens=verify_num_draft_tokens, + gamma=gamma, + ) + accept = AcceptOuts( + correct_len=correct_len, + bonus=bonus, + cap_trim_lens=finalized.cap_trim_lens, + commit_lens=finalized.commit_lens, + new_seq_lens=finalized.new_seq_lens, + out_tokens=out_tokens, + ) + if out is not None: + out.correct_len.copy_(accept.correct_len) + out.bonus.copy_(accept.bonus) + out.cap_trim_lens.copy_(accept.cap_trim_lens) + out.commit_lens.copy_(accept.commit_lens) + out.new_seq_lens.copy_(accept.new_seq_lens) + out.out_tokens.copy_(accept.out_tokens) + return out + return accept + + def sample_simulated_bonus( *, target_logits: torch.Tensor, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index b785bf44e9..6bf618cbc0 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -539,11 +539,6 @@ class DSparkWorkerV2(BaseSpecWorker): noise=self._proposer.noise, greedy_draft=self._greedy_draft, tp_sync=self._tp_sync, - confidence_fn=( - self._verify_planner.compute_confidence_tensor - if self._verify_planner.carries_confidence - else None - ), out=( self._verify_epilogue.draft_tokens_buf if self._verify_epilogue is not None @@ -899,13 +894,6 @@ class DSparkWorkerV2(BaseSpecWorker): coordination_plan.apply(batch, "target", rank, local_only=target_local_only) confidence = proposal.confidence - if confidence is None: - confidence = self._verify_planner.compute_confidence_tensor( - draft_hidden=proposal.draft_hidden, - anchor_tokens=draft_block_ids[:, 0], - draft_tokens=draft_tokens, - confidence_tap=proposal.confidence_tap, - ) verify_token_budget = self._verify_planner.resolve_verify_token_budget( draft_input=draft_input, @@ -939,7 +927,7 @@ class DSparkWorkerV2(BaseSpecWorker): fold_eligible = ( self._verify_executor.verify_epilogue is not None and proposal.folded - # The epilogue's in-graph accept is greedy (accept_greedy_triton); + # The epilogue's in-graph accept is greedy (AcceptGreedy); # sampling batches must take the eager accept path even when the # draft proposal itself folded. and (sampling_info is None or sampling_info.is_all_greedy) @@ -1075,6 +1063,7 @@ class DSparkWorkerV2(BaseSpecWorker): target_logits=logits_output.next_token_logits, layout=layout, confidence=confidence, + confidence_raw=proposal.confidence_raw, prefix_lens=prefix_lens, draft_tokens=draft_tokens, draft_block=draft_block, diff --git a/test/registered/spec/dspark/test_dspark_draft_graph.py b/test/registered/spec/dspark/test_dspark_draft_graph.py index 2024dce253..2375e3cfc7 100644 --- a/test/registered/spec/dspark/test_dspark_draft_graph.py +++ b/test/registered/spec/dspark/test_dspark_draft_graph.py @@ -1,5 +1,5 @@ import unittest -from types import SimpleNamespace +from types import MethodType, SimpleNamespace from unittest import mock import torch @@ -7,7 +7,11 @@ import torch from sglang.kernels.ops.speculative.dspark.dspark_draft_model import ( MarkovGreedyStep, ) -from sglang.srt.models.dspark import VanillaMarkov +from sglang.srt.models.dspark import ( + DSparkConfidenceHead, + DSparkDraftMixin, + VanillaMarkov, +) from sglang.srt.speculative.dspark_components.dspark_draft import ( DraftNoise, draft_markov_block, @@ -127,9 +131,15 @@ class TestDraftMarkovBlockGraph(CustomTestCase): weight = torch.randn(VOCAB, 64, device=DEVICE).to(torch.bfloat16) model = SimpleNamespace( markov_head=head, + confidence_head=DSparkConfidenceHead(hidden_size=64, markov_rank=RANK).to( + DEVICE + ), sample_from_anchor=True, lm_head=SimpleNamespace(org_vocab_size=VOCAB, weight=weight), - compute_base_logits=lambda hidden: (hidden @ weight.T, None), + compute_base_logits=lambda hidden: (hidden @ weight.T, hidden), + ) + model.compute_confidence = MethodType( + DSparkDraftMixin.compute_confidence, model ) tp_sync = SimpleNamespace(sync=lambda site, value: value) noise = DraftNoise( @@ -179,6 +189,14 @@ class TestDraftMarkovBlockGraph(CustomTestCase): ) self.assertTrue(torch.equal(sampler.step_logits[:BS], eager.corrected_logits)) self.assertEqual(noise.offset.item(), eager_noise.offset.item()) + # Folded steps carry the confidence the eager proposal computes. + confidence, confidence_raw = model.compute_confidence( + anchor_tokens=input_ids.view(BS, GAMMA)[:, 0], + draft_tokens=eager.draft_tokens, + confidence_features=hidden, + ) + self.assertTrue(torch.equal(sampler.confidence_out[:BS], confidence)) + self.assertTrue(torch.equal(sampler.confidence_raw_out[:BS], confidence_raw)) if __name__ == "__main__": diff --git a/test/registered/spec/dspark/test_dspark_sts.py b/test/registered/spec/dspark/test_dspark_sts.py index b7b890c65b..ff953ca752 100644 --- a/test/registered/spec/dspark/test_dspark_sts.py +++ b/test/registered/spec/dspark/test_dspark_sts.py @@ -9,7 +9,7 @@ from sglang.benchmark.dspark_sts_fit import ( expected_calibration_error, fit_sts_temperatures, ) -from sglang.srt.models.dspark import DSparkConfidenceHead +from sglang.srt.models.dspark import DSparkConfidenceHead, VanillaMarkov from sglang.srt.speculative.dspark_components.dspark_sts import ( DSparkStsCalibration, StsDataRecorder, @@ -40,11 +40,38 @@ class TestApplySts(CustomTestCase): for got, want in zip(row, expected_row): self.assertAlmostEqual(got, want, places=6) - def test_apply_sts_stashes_raw_logit(self): + +class TestConfidencePredict(CustomTestCase): + def test_predict_applies_sts_to_the_raw_logit(self): + torch.manual_seed(0) head = DSparkConfidenceHead(hidden_size=8, markov_rank=4, with_markov=False) - confidence_raw = torch.randn(2, 5) - head.apply_sts(confidence_raw) - self.assertIs(head._last_confidence_raw, confidence_raw) + head.sts_temperatures = torch.tensor([0.5, 1.0, 2.0]) + confidence, confidence_raw = head.predict( + torch.randn(2 * 3, 8), + anchor_tokens=torch.tensor([1, 2]), + draft_tokens=torch.tensor([[3, 4, 5], [6, 7, 8]]), + markov_head=None, + ) + self.assertEqual(confidence_raw.shape, (2, 3)) + self.assertTrue(torch.equal(confidence, head.apply_sts(confidence_raw))) + + def test_markov_features_are_the_previous_tokens(self): + # Step i reads the embedding of the token before it: the anchor, then drafts. + torch.manual_seed(0) + head = DSparkConfidenceHead(hidden_size=8, markov_rank=4, with_markov=True) + markov_head = VanillaMarkov(vocab_size=16, markov_rank=4) + features = torch.randn(2, 3, 8) + anchor_tokens = torch.tensor([1, 2]) + draft_tokens = torch.tensor([[3, 4, 5], [6, 7, 8]]) + _, confidence_raw = head.predict( + features.view(6, 8), + anchor_tokens=anchor_tokens, + draft_tokens=draft_tokens, + markov_head=markov_head, + ) + prev_tokens = torch.tensor([[1, 3, 4], [2, 6, 7]]) + expected = head(features, markov_head.get_prev_embeddings(prev_tokens)) + self.assertTrue(torch.equal(confidence_raw, expected)) class TestDSparkStsCalibration(CustomTestCase): diff --git a/test/registered/unit/spec/test_dspark_accept_tail.py b/test/registered/unit/spec/test_dspark_accept_tail.py new file mode 100644 index 0000000000..3a9dc7b9d9 --- /dev/null +++ b/test/registered/unit/spec/test_dspark_accept_tail.py @@ -0,0 +1,136 @@ +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.speculative.dspark_components.dspark_draft import DraftBlockResult +from sglang.srt.speculative.dspark_components.dspark_verify import ( + DsparkVerifyEpilogue, + TargetVerifyExecutor, +) +from sglang.srt.speculative.spec_tp_sync import SpecTpSyncSite +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +BS, STRIDE, VOCAB = 3, 5, 50 + + +class _RecordingTpSync: + def __init__(self): + self.calls = [] + + def sync(self, site, values): + self.calls.append((site, values.clone())) + return values + + +def _greedy_inputs(): + torch.manual_seed(0) + candidates = torch.randint(0, VOCAB, (BS, STRIDE)) + target_predict = torch.randint(0, VOCAB, (BS, STRIDE)) + # Rows match the drafts for 0, 2 and all positions. + target_predict[1, :2] = candidates[1, 1:3] + target_predict[2, :-1] = candidates[2, 1:] + logits = torch.randn(BS * STRIDE, VOCAB) + logits.scatter_(1, target_predict.view(-1, 1), 100.0) + prefix_lens = torch.tensor([7, 11, 13], dtype=torch.int64) + return candidates, logits, prefix_lens + + +class TestAcceptTailParity(CustomTestCase): + def _run(self, simulated_correct_len, verify_lens): + candidates, logits, prefix_lens = _greedy_inputs() + draft_tokens = candidates[:, 1:].contiguous() + + graph_sync = _RecordingTpSync() + epilogue = DsparkVerifyEpilogue( + max_bs=BS + 1, + verify_num_draft_tokens=STRIDE, + device="cpu", + tp_sync=graph_sync, + simulated_correct_len=simulated_correct_len, + ) + epilogue._accept( + candidates=candidates, + logits=logits, + draft_tokens=draft_tokens, + seq_lens=prefix_lens, + cutoff_verify_lens=verify_lens, + ) + graph = epilogue.read_accept(BS) + + eager_sync = _RecordingTpSync() + target_worker = SimpleNamespace( + model_runner=SimpleNamespace( + model_config=SimpleNamespace( + hf_text_config=SimpleNamespace(model_type="test") + ) + ) + ) + executor = TargetVerifyExecutor( + target_worker=target_worker, + gamma=STRIDE - 1, + verify_num_draft_tokens=STRIDE, + model_runner=None, + kv_injector=None, + tp_sync=eager_sync, + simulated_correct_len=simulated_correct_len, + ) + eager = executor.accept_and_finalize( + folded_accept=False, + bs=BS, + verify_ids_2d=candidates, + target_logits=logits, + draft_block=DraftBlockResult( + draft_tokens=draft_tokens, + corrected_logits=None, + greedy_mask=torch.ones(BS, dtype=torch.bool), + temperatures=torch.ones(BS), + greedy_draft=True, + ), + sampling_info=None, + draft_input=None, + layout=None + if verify_lens is None + else SimpleNamespace(verify_lens=verify_lens), + prefix_lens=prefix_lens, + draft_tokens=draft_tokens, + ) + + for field in eager.__struct_fields__: + self.assertTrue( + torch.equal( + getattr(graph, field).cpu().long(), + getattr(eager, field).cpu().long(), + ), + field, + ) + self.assertEqual( + [site for site, _ in graph_sync.calls], + [SpecTpSyncSite.DSPARK_ACCEPT_GRAPH] * 3, + ) + self.assertEqual( + [site for site, _ in eager_sync.calls], + [SpecTpSyncSite.DSPARK_ACCEPT_GREEDY] * 3, + ) + for (_, graph_values), (_, eager_values) in zip( + graph_sync.calls, eager_sync.calls + ): + self.assertTrue(torch.equal(graph_values, eager_values)) + + def test_graph_accept_matches_eager_greedy(self): + # Row 1 has a 2-token budget, so its 2 correct drafts are capped to 1. + ragged = torch.tensor([STRIDE, 2, 3], dtype=torch.int64) + for simulated in (None, 1): + for verify_lens in (None, ragged): + with self.subTest(simulated=simulated, ragged=verify_lens is not None): + simulated_correct_len = ( + None if simulated is None else torch.tensor([simulated]) + ) + self._run(simulated_correct_len, verify_lens) + + +if __name__ == "__main__": + unittest.main()