[DSpark] Share draft params, logits dtype, confidence and the accept tail between graph and eager (#43790)
This commit is contained in:
@@ -65,16 +65,11 @@ from sglang.srt.speculative.ragged_verify import (
|
||||
read_ragged_verify_mode,
|
||||
)
|
||||
from sglang.srt.utils import add_prefix, is_npu
|
||||
from sglang.srt.utils.invariants import Bucket, InClosedRange, Invariant, expect
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PAD_NUM_HEADS = 64
|
||||
|
||||
# DSpark confidence is a per-token score that must stay in [0, 1].
|
||||
_CONFIDENCE = Invariant(
|
||||
"dspark.model.confidence", Bucket.GUARD, InClosedRange(0.0, 1.0)
|
||||
)
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
@@ -1002,25 +997,15 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
|
||||
self,
|
||||
*,
|
||||
anchor_tokens: torch.Tensor,
|
||||
sampled_tokens: torch.Tensor,
|
||||
x_post_hc: torch.Tensor,
|
||||
) -> Optional[torch.Tensor]:
|
||||
confidence_head = self.confidence_head
|
||||
if confidence_head is None:
|
||||
return None
|
||||
bs = int(anchor_tokens.shape[0])
|
||||
x_post_hc = x_post_hc.view(bs, self.gamma, -1)
|
||||
if confidence_head.with_markov:
|
||||
prev_seq = torch.cat(
|
||||
[anchor_tokens.view(-1, 1), sampled_tokens[:, : self.gamma - 1]], dim=1
|
||||
)
|
||||
markov_embed_stack = self.markov_head.get_prev_embeddings(prev_seq)
|
||||
else:
|
||||
markov_embed_stack = None
|
||||
confidence_raw = confidence_head(x_post_hc, markov_embed_stack)
|
||||
confidence = confidence_head.apply_sts(confidence_raw)
|
||||
expect(_CONFIDENCE, confidence)
|
||||
return confidence
|
||||
draft_tokens: torch.Tensor,
|
||||
confidence_features: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.confidence_head.predict(
|
||||
confidence_features,
|
||||
anchor_tokens=anchor_tokens,
|
||||
draft_tokens=draft_tokens,
|
||||
markov_head=self.markov_head,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> None:
|
||||
params_dict = dict(self.named_parameters())
|
||||
|
||||
@@ -25,12 +25,18 @@ from sglang.srt.speculative.ragged_verify import (
|
||||
RaggedVerifyMode,
|
||||
read_ragged_verify_mode,
|
||||
)
|
||||
from sglang.srt.utils.invariants import Bucket, InClosedRange, Invariant, expect
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.distributed.device_communicators.vocab_gather import VocabGather
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DSpark confidence is a per-token score that must stay in [0, 1].
|
||||
_CONFIDENCE = Invariant(
|
||||
"dspark.model.confidence", Bucket.GUARD, InClosedRange(0.0, 1.0)
|
||||
)
|
||||
|
||||
|
||||
def gather_and_crop_vocab(
|
||||
local_logits: torch.Tensor, lm_head: nn.Module
|
||||
@@ -299,7 +305,6 @@ class DSparkConfidenceHead(nn.Module):
|
||||
self.register_buffer(
|
||||
"sts_temperatures", torch.ones((), dtype=torch.float32), persistent=False
|
||||
)
|
||||
self._last_confidence_raw: Optional[torch.Tensor] = None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -321,9 +326,30 @@ class DSparkConfidenceHead(nn.Module):
|
||||
return self.proj(features).squeeze(-1)
|
||||
|
||||
def apply_sts(self, confidence_raw: torch.Tensor) -> torch.Tensor:
|
||||
self._last_confidence_raw = confidence_raw
|
||||
return torch.sigmoid(confidence_raw.float() / self.sts_temperatures)
|
||||
|
||||
def predict(
|
||||
self,
|
||||
features: torch.Tensor,
|
||||
*,
|
||||
anchor_tokens: torch.Tensor,
|
||||
draft_tokens: torch.Tensor,
|
||||
markov_head: BaseDSparkMarkovHead,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""The [bs, gamma] confidence of a drafted block and its raw logit."""
|
||||
bs, gamma = draft_tokens.shape
|
||||
if self.with_markov:
|
||||
prev_tokens = torch.cat(
|
||||
[anchor_tokens.view(-1, 1), draft_tokens[:, : gamma - 1]], dim=1
|
||||
)
|
||||
markov_embed_stack = markov_head.get_prev_embeddings(prev_tokens)
|
||||
else:
|
||||
markov_embed_stack = None
|
||||
confidence_raw = self(features.view(bs, gamma, -1), markov_embed_stack)
|
||||
confidence = self.apply_sts(confidence_raw)
|
||||
expect(_CONFIDENCE, confidence)
|
||||
return confidence, confidence_raw
|
||||
|
||||
|
||||
def build_confidence_head(config) -> Optional[nn.Module]:
|
||||
if read_ragged_verify_mode() is RaggedVerifyMode.STATIC:
|
||||
@@ -399,8 +425,9 @@ class DSparkDraftMixin:
|
||||
|
||||
def compute_base_logits(
|
||||
self, hidden: torch.Tensor
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Project the draft's raw final hidden through the target lm_head.
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Project the draft's raw final hidden through the target lm_head;
|
||||
also returns the features the confidence head reads.
|
||||
|
||||
muP targets (Inkling) train the draft against a FOLDED head (weights
|
||||
pre-divided by logits_mup_width_multiplier) while serving attaches the
|
||||
@@ -414,11 +441,26 @@ class DSparkDraftMixin:
|
||||
"DSpark dense draft requires the target lm_head "
|
||||
"(call attach_shared_modules first)."
|
||||
)
|
||||
confidence_features = hidden
|
||||
if self.logits_mup_width_multiplier:
|
||||
hidden = hidden / self.logits_mup_width_multiplier
|
||||
local_logits = project_through_lm_head(hidden, self.lm_head)
|
||||
base_logits = gather_and_crop_vocab(local_logits, self.lm_head)
|
||||
return base_logits, None
|
||||
return base_logits, confidence_features
|
||||
|
||||
def compute_confidence(
|
||||
self,
|
||||
*,
|
||||
anchor_tokens: torch.Tensor,
|
||||
draft_tokens: torch.Tensor,
|
||||
confidence_features: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.confidence_head.predict(
|
||||
confidence_features,
|
||||
anchor_tokens=anchor_tokens,
|
||||
draft_tokens=draft_tokens,
|
||||
markov_head=self.markov_head,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
markov_weights = []
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
from typing import TYPE_CHECKING, NamedTuple, Optional
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -72,9 +73,9 @@ class DraftForwardResult(msgspec.Struct, frozen=True):
|
||||
class DraftProposal(msgspec.Struct, frozen=True):
|
||||
draft_block_ids: torch.Tensor
|
||||
draft_block: DraftBlockResult
|
||||
draft_hidden: Optional[torch.Tensor]
|
||||
# [bs, gamma]; None when the draft has no confidence head.
|
||||
confidence: Optional[torch.Tensor] = None
|
||||
confidence_tap: Optional[torch.Tensor] = None
|
||||
confidence_raw: Optional[torch.Tensor] = None
|
||||
folded: bool = False
|
||||
|
||||
|
||||
@@ -118,6 +119,51 @@ def resolve_greedy_mask(
|
||||
return (sampling_info.top_ks <= 1).view(-1)
|
||||
|
||||
|
||||
class DraftParams(NamedTuple):
|
||||
temperatures: torch.Tensor
|
||||
greedy_mask: torch.Tensor
|
||||
# The temperature each step samples at: 0 on argmax rows.
|
||||
step_temperatures: torch.Tensor
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _zero(device: torch.device) -> torch.Tensor:
|
||||
return torch.zeros((), dtype=torch.float32, device=device)
|
||||
|
||||
|
||||
def resolve_draft_params(
|
||||
*,
|
||||
sampling_info,
|
||||
bs: int,
|
||||
device: Optional[torch.device] = None,
|
||||
out: Optional[DraftParams] = None,
|
||||
) -> DraftParams:
|
||||
"""Written into the ``[:bs]`` rows of ``out`` when given (the graph's
|
||||
static buffers), else into fresh tensors on ``device``."""
|
||||
if out is not None:
|
||||
device = out.temperatures.device
|
||||
else:
|
||||
out = DraftParams(
|
||||
temperatures=torch.empty(bs, dtype=torch.float32, device=device),
|
||||
greedy_mask=torch.empty(bs, dtype=torch.bool, device=device),
|
||||
step_temperatures=torch.empty(bs, dtype=torch.float32, device=device),
|
||||
)
|
||||
temperatures, greedy_mask, step_temperatures = (buf[:bs] for buf in out)
|
||||
if sampling_info is None:
|
||||
temperatures.fill_(1.0)
|
||||
greedy_mask.fill_(True)
|
||||
step_temperatures.zero_()
|
||||
else:
|
||||
torch.clamp(
|
||||
sampling_info.temperatures.view(-1)[:bs].to(torch.float32),
|
||||
min=1e-5,
|
||||
out=temperatures,
|
||||
)
|
||||
torch.le(sampling_info.top_ks.view(-1)[:bs], 1, out=greedy_mask)
|
||||
torch.where(greedy_mask, _zero(device), temperatures, out=step_temperatures)
|
||||
return DraftParams(temperatures, greedy_mask, step_temperatures)
|
||||
|
||||
|
||||
# XORed into the server seed so the draft noise differs from other samplers'.
|
||||
_DRAFT_NOISE_DOMAIN = 0x2D2A9C317E44B1F3
|
||||
|
||||
@@ -234,14 +280,6 @@ def draft_markov_block(
|
||||
return torch.stack(draft_tokens, dim=1), logits_out
|
||||
|
||||
|
||||
def resolve_draft_temperatures(
|
||||
*, bs: int, sampling_info, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
if sampling_info is None:
|
||||
return torch.ones(bs, dtype=torch.float32, device=device)
|
||||
return sampling_info.temperatures.view(-1).to(torch.float32).clamp_min(1e-5)
|
||||
|
||||
|
||||
def sample_draft_block(
|
||||
*,
|
||||
base_logits: torch.Tensor,
|
||||
@@ -254,17 +292,15 @@ def sample_draft_block(
|
||||
tp_sync: SpecTpSync,
|
||||
greedy_draft: bool,
|
||||
) -> DraftBlockResult:
|
||||
bs = base_logits.shape[0]
|
||||
greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device)
|
||||
temperatures = resolve_draft_temperatures(
|
||||
bs=bs, sampling_info=sampling_info, device=device
|
||||
params = resolve_draft_params(
|
||||
sampling_info=sampling_info, bs=base_logits.shape[0], device=device
|
||||
)
|
||||
draft_tokens, step_logits = draft_markov_block(
|
||||
markov_head=markov_head,
|
||||
base_logits=base_logits,
|
||||
anchor_tokens=anchor_tokens,
|
||||
draft_hidden=draft_hidden,
|
||||
step_temperatures=temperatures.masked_fill(greedy_mask, 0.0),
|
||||
step_temperatures=params.step_temperatures,
|
||||
noise=noise,
|
||||
tp_sync=tp_sync,
|
||||
greedy_draft=greedy_draft,
|
||||
@@ -274,8 +310,8 @@ def sample_draft_block(
|
||||
corrected_logits=(
|
||||
None if greedy_draft else expect(_DRAFT_STEP_LOGITS, step_logits)
|
||||
),
|
||||
greedy_mask=greedy_mask,
|
||||
temperatures=temperatures,
|
||||
greedy_mask=params.greedy_mask,
|
||||
temperatures=params.temperatures,
|
||||
greedy_draft=greedy_draft,
|
||||
)
|
||||
|
||||
@@ -359,8 +395,7 @@ class DraftBlockProposer:
|
||||
)
|
||||
draft_block_ids = fwd.draft_block_ids
|
||||
|
||||
folded_confidence = None
|
||||
confidence_tap = None
|
||||
confidence = confidence_raw = None
|
||||
folded = False
|
||||
if (
|
||||
envs.SGLANG_DSPARK_FOLDED_PROPOSAL.get()
|
||||
@@ -373,15 +408,16 @@ class DraftBlockProposer:
|
||||
corrected_logits=(
|
||||
None if self._greedy_draft else draft_sampler.step_logits[:bs]
|
||||
),
|
||||
greedy_mask=draft_sampler.greedy_mask[:bs],
|
||||
temperatures=draft_sampler.temperatures[:bs],
|
||||
greedy_mask=draft_sampler.params.greedy_mask[:bs],
|
||||
temperatures=draft_sampler.params.temperatures[:bs],
|
||||
greedy_draft=self._greedy_draft,
|
||||
)
|
||||
if draft_sampler.confidence_out is not None:
|
||||
folded_confidence = draft_sampler.confidence_out[:bs]
|
||||
confidence = draft_sampler.confidence_out[:bs]
|
||||
confidence_raw = draft_sampler.confidence_raw_out[:bs]
|
||||
else:
|
||||
with self._base_logits_context():
|
||||
base_logits, confidence_tap = self.draft_model.compute_base_logits(
|
||||
base_logits, confidence_features = self.draft_model.compute_base_logits(
|
||||
fwd.raw_hidden
|
||||
)
|
||||
base_logits = base_logits.view(bs, self.gamma, -1)
|
||||
@@ -396,6 +432,12 @@ class DraftBlockProposer:
|
||||
tp_sync=self._tp_sync,
|
||||
greedy_draft=self._greedy_draft,
|
||||
)
|
||||
if self.draft_model.confidence_head is not None:
|
||||
confidence, confidence_raw = self.draft_model.compute_confidence(
|
||||
anchor_tokens=draft_block_ids[:, 0],
|
||||
draft_tokens=draft_block.draft_tokens,
|
||||
confidence_features=confidence_features,
|
||||
)
|
||||
proposal_block_ids = (
|
||||
draft_block_ids
|
||||
if self.sample_from_anchor
|
||||
@@ -404,9 +446,8 @@ class DraftBlockProposer:
|
||||
return DraftProposal(
|
||||
draft_block_ids=proposal_block_ids,
|
||||
draft_block=draft_block,
|
||||
draft_hidden=fwd.draft_hidden_3d,
|
||||
confidence=folded_confidence,
|
||||
confidence_tap=confidence_tap,
|
||||
confidence=confidence,
|
||||
confidence_raw=confidence_raw,
|
||||
folded=folded,
|
||||
)
|
||||
|
||||
|
||||
@@ -7,7 +7,9 @@ import torch
|
||||
|
||||
from sglang.srt.speculative.dspark_components.dspark_draft import (
|
||||
DraftNoise,
|
||||
DraftParams,
|
||||
draft_markov_block,
|
||||
resolve_draft_params,
|
||||
select_draft_hidden_without_anchor,
|
||||
step_logits_width,
|
||||
)
|
||||
@@ -29,7 +31,6 @@ class DsparkDraftSampler:
|
||||
noise: DraftNoise,
|
||||
tp_sync: SpecTpSync,
|
||||
greedy_draft: bool,
|
||||
confidence_fn=None,
|
||||
out=None,
|
||||
):
|
||||
self.model = model
|
||||
@@ -46,55 +47,47 @@ class DsparkDraftSampler:
|
||||
self.out = torch.empty(
|
||||
(self.max_bs * self.gamma,), dtype=torch.int64, device=device
|
||||
)
|
||||
self.confidence_fn = confidence_fn
|
||||
self.confidence_out = (
|
||||
torch.empty((self.max_bs, self.gamma), dtype=torch.float32, device=device)
|
||||
if confidence_fn is not None
|
||||
else None
|
||||
)
|
||||
confidence_head = model.confidence_head
|
||||
if confidence_head is None:
|
||||
self.confidence_out = self.confidence_raw_out = None
|
||||
else:
|
||||
self.confidence_out = torch.empty(
|
||||
(self.max_bs, self.gamma), dtype=torch.float32, device=device
|
||||
)
|
||||
self.confidence_raw_out = torch.empty(
|
||||
(self.max_bs, self.gamma),
|
||||
dtype=confidence_head.proj.weight.dtype,
|
||||
device=device,
|
||||
)
|
||||
self.greedy_draft = greedy_draft
|
||||
self._noise = noise
|
||||
self._tp_sync = tp_sync
|
||||
# Staged before each replay; step_temperatures is 0 on argmax rows.
|
||||
self.temperatures = torch.ones(
|
||||
(self.max_bs,), dtype=torch.float32, device=device
|
||||
# Staged before each replay.
|
||||
self.params = DraftParams(
|
||||
temperatures=torch.ones((self.max_bs,), dtype=torch.float32, device=device),
|
||||
greedy_mask=torch.ones((self.max_bs,), dtype=torch.bool, device=device),
|
||||
step_temperatures=torch.zeros(
|
||||
(self.max_bs,), dtype=torch.float32, device=device
|
||||
),
|
||||
)
|
||||
self.greedy_mask = torch.ones((self.max_bs,), dtype=torch.bool, device=device)
|
||||
self.step_temperatures = torch.zeros(
|
||||
(self.max_bs,), dtype=torch.float32, device=device
|
||||
)
|
||||
self._zero = torch.zeros((), dtype=torch.float32, device=device)
|
||||
# [max_bs, gamma, vocab] step logits for the sampling accept; a greedy
|
||||
# draft keeps none.
|
||||
self.step_logits = (
|
||||
None
|
||||
if greedy_draft
|
||||
else torch.empty(
|
||||
(self.max_bs, self.gamma, _step_logits_width(model)),
|
||||
dtype=_step_logits_dtype(model),
|
||||
device=device,
|
||||
# [max_bs, gamma, vocab] step logits for the sampling accept, sized on
|
||||
# the first call; a greedy draft keeps none.
|
||||
self.step_logits: Optional[torch.Tensor] = None
|
||||
|
||||
def _step_logits(self, base_logits: torch.Tensor) -> torch.Tensor:
|
||||
if self.step_logits is None:
|
||||
# The runner's warmup calls the tail eagerly before capture.
|
||||
assert not torch.cuda.is_current_stream_capturing()
|
||||
self.step_logits = torch.empty(
|
||||
(self.max_bs, self.gamma, _step_logits_width(self.model)),
|
||||
dtype=self.markov_head.step_logits_dtype(base_logits.dtype),
|
||||
device=base_logits.device,
|
||||
)
|
||||
)
|
||||
return self.step_logits
|
||||
|
||||
def stage_sampling_params(self, *, bs: int, sampling_info) -> None:
|
||||
"""Must run before the draft graph replay that reads these buffers."""
|
||||
if sampling_info is None:
|
||||
self.temperatures[:bs].fill_(1.0)
|
||||
self.greedy_mask[:bs].fill_(True)
|
||||
self.step_temperatures[:bs].zero_()
|
||||
return
|
||||
torch.clamp(
|
||||
sampling_info.temperatures.view(-1)[:bs].to(torch.float32),
|
||||
min=1e-5,
|
||||
out=self.temperatures[:bs],
|
||||
)
|
||||
torch.le(sampling_info.top_ks.view(-1)[:bs], 1, out=self.greedy_mask[:bs])
|
||||
torch.where(
|
||||
self.greedy_mask[:bs],
|
||||
self._zero,
|
||||
self.temperatures[:bs],
|
||||
out=self.step_temperatures[:bs],
|
||||
)
|
||||
resolve_draft_params(sampling_info=sampling_info, bs=bs, out=self.params)
|
||||
|
||||
def __call__(self, hidden_states, input_ids):
|
||||
bs = hidden_states.shape[0] // self.query_token_num
|
||||
@@ -107,7 +100,7 @@ class DsparkDraftSampler:
|
||||
bs=bs,
|
||||
gamma=self.gamma,
|
||||
)
|
||||
base_logits, confidence_tap = self.model.compute_base_logits(model_hidden)
|
||||
base_logits, confidence_features = self.model.compute_base_logits(model_hidden)
|
||||
base_logits = base_logits.view(bs, self.gamma, -1)
|
||||
anchor = input_ids.view(bs, self.query_token_num)[:, 0]
|
||||
|
||||
@@ -116,21 +109,23 @@ class DsparkDraftSampler:
|
||||
base_logits=base_logits,
|
||||
anchor_tokens=anchor,
|
||||
draft_hidden=sample_hidden,
|
||||
step_temperatures=self.step_temperatures[:bs],
|
||||
step_temperatures=self.params.step_temperatures[:bs],
|
||||
noise=self._noise,
|
||||
tp_sync=self._tp_sync,
|
||||
logits_out=None if self.greedy_draft else self.step_logits[:bs],
|
||||
logits_out=None
|
||||
if self.greedy_draft
|
||||
else self._step_logits(base_logits)[:bs],
|
||||
greedy_draft=self.greedy_draft,
|
||||
)
|
||||
self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1))
|
||||
if self.confidence_out is not None:
|
||||
confidence = self.confidence_fn(
|
||||
draft_hidden=sample_hidden,
|
||||
confidence, confidence_raw = self.model.compute_confidence(
|
||||
anchor_tokens=anchor,
|
||||
draft_tokens=draft_tokens,
|
||||
confidence_tap=confidence_tap,
|
||||
confidence_features=confidence_features,
|
||||
)
|
||||
self.confidence_out[:bs].copy_(confidence)
|
||||
self.confidence_raw_out[:bs].copy_(confidence_raw)
|
||||
|
||||
|
||||
def _step_logits_width(model) -> int:
|
||||
@@ -138,16 +133,6 @@ def _step_logits_width(model) -> int:
|
||||
return step_logits_width(model.markov_head, int(model.lm_head.org_vocab_size))
|
||||
|
||||
|
||||
def _step_logits_dtype(model) -> torch.dtype:
|
||||
# A quantized head's packed weight has no logits dtype; it emits the draft's.
|
||||
weight = model.lm_head.weight
|
||||
if weight.is_floating_point():
|
||||
base_dtype = weight.dtype
|
||||
else:
|
||||
base_dtype = next(model.markov_head.parameters()).dtype
|
||||
return model.markov_head.step_logits_dtype(base_dtype)
|
||||
|
||||
|
||||
def maybe_build_draft_sampler(
|
||||
*,
|
||||
draft_model,
|
||||
@@ -158,7 +143,6 @@ def maybe_build_draft_sampler(
|
||||
noise: DraftNoise,
|
||||
greedy_draft: bool,
|
||||
tp_sync: SpecTpSync,
|
||||
confidence_fn=None,
|
||||
out=None,
|
||||
) -> Optional[DsparkDraftSampler]:
|
||||
"""Build the graph-folded draft sampler, or None (reason logged) when the
|
||||
@@ -186,6 +170,5 @@ def maybe_build_draft_sampler(
|
||||
noise=noise,
|
||||
tp_sync=tp_sync,
|
||||
greedy_draft=greedy_draft,
|
||||
confidence_fn=confidence_fn,
|
||||
out=out,
|
||||
)
|
||||
|
||||
@@ -824,6 +824,7 @@ class DsparkStepObservers:
|
||||
target_logits: Optional[torch.Tensor],
|
||||
layout,
|
||||
confidence: Optional[torch.Tensor],
|
||||
confidence_raw: Optional[torch.Tensor],
|
||||
prefix_lens: torch.Tensor,
|
||||
draft_tokens: torch.Tensor,
|
||||
draft_block,
|
||||
@@ -838,20 +839,20 @@ class DsparkStepObservers:
|
||||
dp_tier_num_tokens: Optional[int],
|
||||
) -> None:
|
||||
planner = self._planner
|
||||
if not proposal_folded:
|
||||
self._maybe_record_sts_collect(
|
||||
verify_ids_2d=verify_ids_2d,
|
||||
target_logits=target_logits,
|
||||
bs=bs,
|
||||
)
|
||||
self._confidence_probe.maybe_observe(
|
||||
carries_confidence=planner.carries_confidence,
|
||||
is_compact_mode=planner.is_compact_mode,
|
||||
confidence_raw=planner.last_confidence_raw,
|
||||
verify_ids_2d=verify_ids_2d,
|
||||
target_logits=target_logits,
|
||||
bs=bs,
|
||||
)
|
||||
self._maybe_record_sts_collect(
|
||||
confidence_raw=confidence_raw,
|
||||
verify_ids_2d=verify_ids_2d,
|
||||
target_logits=target_logits,
|
||||
bs=bs,
|
||||
)
|
||||
self._confidence_probe.maybe_observe(
|
||||
carries_confidence=planner.carries_confidence,
|
||||
is_compact_mode=planner.is_compact_mode,
|
||||
confidence_raw=confidence_raw,
|
||||
verify_ids_2d=verify_ids_2d,
|
||||
target_logits=target_logits,
|
||||
bs=bs,
|
||||
)
|
||||
if self._block_accept_recorder is not None and not proposal_folded:
|
||||
self._block_accept_recorder.observe_verify_step(
|
||||
forward_ct=forward_ct,
|
||||
@@ -929,15 +930,13 @@ class DsparkStepObservers:
|
||||
def _maybe_record_sts_collect(
|
||||
self,
|
||||
*,
|
||||
confidence_raw: Optional[torch.Tensor],
|
||||
verify_ids_2d: torch.Tensor,
|
||||
target_logits: Optional[torch.Tensor],
|
||||
bs: int,
|
||||
) -> None:
|
||||
if not self._sts_collect_path:
|
||||
return
|
||||
if not self._planner.carries_confidence:
|
||||
return
|
||||
confidence_raw = self._planner.last_confidence_raw
|
||||
if confidence_raw is None:
|
||||
return
|
||||
if self._sts_recorder is None:
|
||||
|
||||
@@ -45,7 +45,6 @@ from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite
|
||||
from sglang.srt.utils.common import require_mlp_tp_gather
|
||||
from sglang.srt.utils.invariants import (
|
||||
Bucket,
|
||||
InClosedRange,
|
||||
Invariant,
|
||||
IsTrue,
|
||||
expect,
|
||||
@@ -54,10 +53,6 @@ from sglang.srt.utils.invariants import (
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DSpark confidence is a per-token score that must stay in [0, 1].
|
||||
_CONFIDENCE = Invariant(
|
||||
"dspark.planner.confidence", Bucket.GUARD, InClosedRange(0.0, 1.0)
|
||||
)
|
||||
# Scheduled verify lengths must not exceed the per-step token budget.
|
||||
_VERIFY_LEN_BUDGET = Invariant("dspark.verify_len_budget", Bucket.GUARD, IsTrue())
|
||||
|
||||
@@ -214,12 +209,6 @@ class DSparkVerifyPlanner:
|
||||
def carries_confidence(self) -> bool:
|
||||
return self._confidence_head is not None
|
||||
|
||||
@property
|
||||
def last_confidence_raw(self) -> Optional[torch.Tensor]:
|
||||
if self._confidence_head is None:
|
||||
return None
|
||||
return self._confidence_head._last_confidence_raw
|
||||
|
||||
@property
|
||||
def schedules_verify_budget(self) -> bool:
|
||||
return self._budget_planner is not None
|
||||
@@ -252,37 +241,6 @@ class DSparkVerifyPlanner:
|
||||
self._ragged_verify_mode is RaggedVerifyMode.COMPACT and layout is not None
|
||||
)
|
||||
|
||||
def compute_confidence_tensor(
|
||||
self,
|
||||
*,
|
||||
draft_hidden: Optional[torch.Tensor],
|
||||
anchor_tokens: torch.Tensor,
|
||||
draft_tokens: torch.Tensor,
|
||||
confidence_tap: Optional[torch.Tensor] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
if self._confidence_head is None:
|
||||
return None
|
||||
compute_confidence_hook = getattr(self.draft_model, "compute_confidence", None)
|
||||
if compute_confidence_hook is not None:
|
||||
assert confidence_tap is not None, (
|
||||
"dsv4 compute_confidence needs the compute_base_logits tap"
|
||||
)
|
||||
with torch.inference_mode():
|
||||
return compute_confidence_hook(
|
||||
anchor_tokens=anchor_tokens,
|
||||
sampled_tokens=draft_tokens,
|
||||
x_post_hc=confidence_tap,
|
||||
)
|
||||
assert draft_hidden is not None
|
||||
return compute_confidence(
|
||||
draft_hidden=draft_hidden,
|
||||
anchor_tokens=anchor_tokens,
|
||||
draft_tokens=draft_tokens,
|
||||
confidence_head=self._confidence_head,
|
||||
markov_head=self.draft_model.markov_head,
|
||||
gamma=self.gamma,
|
||||
)
|
||||
|
||||
def prepare_verify_budget(
|
||||
self, batch: ScheduleBatch, future_map: FutureMap
|
||||
) -> None:
|
||||
@@ -877,44 +835,6 @@ def apply_logits_adjustments_strided(
|
||||
)
|
||||
|
||||
|
||||
def build_markov_embed_stack(
|
||||
*,
|
||||
anchor_tokens: torch.Tensor,
|
||||
draft_tokens: torch.Tensor,
|
||||
markov_head,
|
||||
gamma: int,
|
||||
) -> torch.Tensor:
|
||||
prev_seq = torch.cat(
|
||||
[anchor_tokens.view(-1, 1), draft_tokens[:, : gamma - 1]], dim=1
|
||||
)
|
||||
return markov_head.get_prev_embeddings(prev_seq)
|
||||
|
||||
|
||||
def compute_confidence(
|
||||
*,
|
||||
draft_hidden: torch.Tensor,
|
||||
anchor_tokens: torch.Tensor,
|
||||
draft_tokens: torch.Tensor,
|
||||
confidence_head,
|
||||
markov_head,
|
||||
gamma: int,
|
||||
) -> torch.Tensor:
|
||||
assert confidence_head is not None
|
||||
if confidence_head.with_markov:
|
||||
markov_embed_stack = build_markov_embed_stack(
|
||||
anchor_tokens=anchor_tokens,
|
||||
draft_tokens=draft_tokens,
|
||||
markov_head=markov_head,
|
||||
gamma=gamma,
|
||||
)
|
||||
else:
|
||||
markov_embed_stack = None
|
||||
confidence_raw = confidence_head(draft_hidden, markov_embed_stack)
|
||||
confidence = confidence_head.apply_sts(confidence_raw)
|
||||
expect(_CONFIDENCE, confidence)
|
||||
return confidence
|
||||
|
||||
|
||||
class DSparkScheduleConfig(msgspec.Struct):
|
||||
gamma: int
|
||||
min_verify_len: int = 1
|
||||
|
||||
@@ -14,8 +14,6 @@ from sglang.kernels.ops.speculative.dspark.dspark_accept import (
|
||||
FinalizeAcceptLens,
|
||||
SelectMixedAccept,
|
||||
SoftmaxTemp,
|
||||
accept_greedy_triton,
|
||||
finalize_accept_lens_triton,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_verify_window import (
|
||||
BuildCommitInjectLayout,
|
||||
@@ -183,46 +181,24 @@ class TargetVerifyExecutor:
|
||||
fused_argmax=self._target_is_dsv41,
|
||||
block_verification=self._block_verification,
|
||||
)
|
||||
if simulate:
|
||||
correct_len = self._simulated_correct_len.repeat(bs).to(correct_len.dtype)
|
||||
if simulate_bonus_sampling_info is not None:
|
||||
bonus = sample_simulated_bonus(
|
||||
target_logits=target_logits,
|
||||
correct_len=correct_len,
|
||||
greedy_bonus=bonus,
|
||||
sampling_info=simulate_bonus_sampling_info,
|
||||
bs=bs,
|
||||
verify_num_draft_tokens=self.verify_num_draft_tokens,
|
||||
)
|
||||
|
||||
site = (
|
||||
sync_site = (
|
||||
SpecTpSyncSite.DSPARK_ACCEPT_GREEDY
|
||||
if accept_sampling_info is None or accept_sampling_info.is_all_greedy
|
||||
else SpecTpSyncSite.DSPARK_ACCEPT_SAMPLE
|
||||
)
|
||||
self._tp_sync.sync(site, correct_len)
|
||||
self._tp_sync.sync(site, bonus)
|
||||
self._tp_sync.sync(site, cap_trim_lens)
|
||||
|
||||
finalized = FinalizeAcceptLens.execute(
|
||||
return build_accept_outs(
|
||||
correct_len=correct_len,
|
||||
bonus=bonus,
|
||||
cap_trim_lens=cap_trim_lens,
|
||||
prefix_lens=prefix_lens,
|
||||
)
|
||||
out_tokens = BuildOutTokens.execute(
|
||||
draft_tokens=draft_tokens,
|
||||
correct_len=correct_len,
|
||||
bonus=bonus,
|
||||
verify_num_draft_tokens=self.verify_num_draft_tokens,
|
||||
gamma=self.gamma,
|
||||
)
|
||||
return AcceptOuts(
|
||||
correct_len=correct_len,
|
||||
bonus=bonus,
|
||||
cap_trim_lens=finalized.cap_trim_lens,
|
||||
commit_lens=finalized.commit_lens,
|
||||
new_seq_lens=finalized.new_seq_lens,
|
||||
out_tokens=out_tokens,
|
||||
verify_num_draft_tokens=self.verify_num_draft_tokens,
|
||||
tp_sync=self._tp_sync,
|
||||
sync_site=sync_site,
|
||||
simulated_correct_len=self._simulated_correct_len,
|
||||
target_logits=target_logits,
|
||||
simulate_bonus_sampling_info=simulate_bonus_sampling_info,
|
||||
)
|
||||
|
||||
def begin_simulated_step(self) -> None:
|
||||
@@ -768,38 +744,27 @@ class DsparkVerifyEpilogue:
|
||||
self, *, candidates, logits, draft_tokens, seq_lens, cutoff_verify_lens=None
|
||||
) -> torch.Tensor:
|
||||
bs = candidates.shape[0]
|
||||
correct_len, bonus, cap_trim_lens = accept_greedy_triton(
|
||||
correct_len, bonus, cap_trim_lens = AcceptGreedy.execute(
|
||||
candidates=candidates,
|
||||
target_logits=logits,
|
||||
verify_num_draft_tokens=self.stride,
|
||||
cutoff_verify_lens=cutoff_verify_lens,
|
||||
fused_argmax=self._fused_argmax,
|
||||
)
|
||||
if self.simulated_correct_len is not None:
|
||||
# The accept runs as served; only its length is replaced.
|
||||
correct_len = self.simulated_correct_len.repeat(bs).to(correct_len.dtype)
|
||||
self._tp_sync.sync(SpecTpSyncSite.DSPARK_ACCEPT_GRAPH, correct_len)
|
||||
self._tp_sync.sync(SpecTpSyncSite.DSPARK_ACCEPT_GRAPH, bonus)
|
||||
self._tp_sync.sync(SpecTpSyncSite.DSPARK_ACCEPT_GRAPH, cap_trim_lens)
|
||||
finalized = finalize_accept_lens_triton(
|
||||
correct_len=correct_len,
|
||||
cap_trim_lens=cap_trim_lens,
|
||||
prefix_lens=seq_lens[:bs],
|
||||
)
|
||||
out_tokens = BuildOutTokens.execute(
|
||||
draft_tokens=draft_tokens,
|
||||
accept = build_accept_outs(
|
||||
correct_len=correct_len,
|
||||
bonus=bonus,
|
||||
verify_num_draft_tokens=self.stride,
|
||||
cap_trim_lens=cap_trim_lens,
|
||||
prefix_lens=seq_lens[:bs],
|
||||
draft_tokens=draft_tokens,
|
||||
gamma=self.gamma,
|
||||
verify_num_draft_tokens=self.stride,
|
||||
tp_sync=self._tp_sync,
|
||||
sync_site=SpecTpSyncSite.DSPARK_ACCEPT_GRAPH,
|
||||
simulated_correct_len=self.simulated_correct_len,
|
||||
out=self.read_accept(bs),
|
||||
)
|
||||
self.correct_len_buf[:bs].copy_(correct_len)
|
||||
self.bonus_buf[:bs].copy_(bonus)
|
||||
self.cap_trim_lens_buf[:bs].copy_(cap_trim_lens.to(torch.int32))
|
||||
self.commit_lens_buf[:bs].copy_(finalized.commit_lens)
|
||||
self.new_seq_lens_buf[:bs].copy_(finalized.new_seq_lens)
|
||||
self.out_tokens_buf[:bs].copy_(out_tokens.view(bs, self.stride))
|
||||
return finalized.commit_lens
|
||||
return accept.commit_lens
|
||||
|
||||
def _commit_inject(
|
||||
self, commit_lens, verify_lens, seq_lens, req_pool_indices, bs: int
|
||||
@@ -838,6 +803,72 @@ class DsparkVerifyEpilogue:
|
||||
)
|
||||
|
||||
|
||||
def build_accept_outs(
|
||||
*,
|
||||
correct_len: torch.Tensor,
|
||||
bonus: torch.Tensor,
|
||||
cap_trim_lens: torch.Tensor,
|
||||
prefix_lens: torch.Tensor,
|
||||
draft_tokens: torch.Tensor,
|
||||
gamma: int,
|
||||
verify_num_draft_tokens: int,
|
||||
tp_sync: SpecTpSync,
|
||||
sync_site: SpecTpSyncSite,
|
||||
simulated_correct_len: Optional[torch.Tensor] = None,
|
||||
target_logits: Optional[torch.Tensor] = None,
|
||||
simulate_bonus_sampling_info=None,
|
||||
out: Optional[AcceptOuts] = None,
|
||||
) -> AcceptOuts:
|
||||
"""``out``: static buffers (the in-graph accept's) to copy the results into."""
|
||||
bs = correct_len.shape[0]
|
||||
if simulated_correct_len is not None:
|
||||
# The accept runs as served; only its length is replaced.
|
||||
correct_len = simulated_correct_len.repeat(bs).to(correct_len.dtype)
|
||||
if simulate_bonus_sampling_info is not None:
|
||||
assert target_logits is not None
|
||||
bonus = sample_simulated_bonus(
|
||||
target_logits=target_logits,
|
||||
correct_len=correct_len,
|
||||
greedy_bonus=bonus,
|
||||
sampling_info=simulate_bonus_sampling_info,
|
||||
bs=bs,
|
||||
verify_num_draft_tokens=verify_num_draft_tokens,
|
||||
)
|
||||
tp_sync.sync(sync_site, correct_len)
|
||||
tp_sync.sync(sync_site, bonus)
|
||||
tp_sync.sync(sync_site, cap_trim_lens)
|
||||
|
||||
finalized = FinalizeAcceptLens.execute(
|
||||
correct_len=correct_len,
|
||||
cap_trim_lens=cap_trim_lens,
|
||||
prefix_lens=prefix_lens,
|
||||
)
|
||||
out_tokens = BuildOutTokens.execute(
|
||||
draft_tokens=draft_tokens,
|
||||
correct_len=correct_len,
|
||||
bonus=bonus,
|
||||
verify_num_draft_tokens=verify_num_draft_tokens,
|
||||
gamma=gamma,
|
||||
)
|
||||
accept = AcceptOuts(
|
||||
correct_len=correct_len,
|
||||
bonus=bonus,
|
||||
cap_trim_lens=finalized.cap_trim_lens,
|
||||
commit_lens=finalized.commit_lens,
|
||||
new_seq_lens=finalized.new_seq_lens,
|
||||
out_tokens=out_tokens,
|
||||
)
|
||||
if out is not None:
|
||||
out.correct_len.copy_(accept.correct_len)
|
||||
out.bonus.copy_(accept.bonus)
|
||||
out.cap_trim_lens.copy_(accept.cap_trim_lens)
|
||||
out.commit_lens.copy_(accept.commit_lens)
|
||||
out.new_seq_lens.copy_(accept.new_seq_lens)
|
||||
out.out_tokens.copy_(accept.out_tokens)
|
||||
return out
|
||||
return accept
|
||||
|
||||
|
||||
def sample_simulated_bonus(
|
||||
*,
|
||||
target_logits: torch.Tensor,
|
||||
|
||||
@@ -539,11 +539,6 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
noise=self._proposer.noise,
|
||||
greedy_draft=self._greedy_draft,
|
||||
tp_sync=self._tp_sync,
|
||||
confidence_fn=(
|
||||
self._verify_planner.compute_confidence_tensor
|
||||
if self._verify_planner.carries_confidence
|
||||
else None
|
||||
),
|
||||
out=(
|
||||
self._verify_epilogue.draft_tokens_buf
|
||||
if self._verify_epilogue is not None
|
||||
@@ -899,13 +894,6 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
coordination_plan.apply(batch, "target", rank, local_only=target_local_only)
|
||||
|
||||
confidence = proposal.confidence
|
||||
if confidence is None:
|
||||
confidence = self._verify_planner.compute_confidence_tensor(
|
||||
draft_hidden=proposal.draft_hidden,
|
||||
anchor_tokens=draft_block_ids[:, 0],
|
||||
draft_tokens=draft_tokens,
|
||||
confidence_tap=proposal.confidence_tap,
|
||||
)
|
||||
|
||||
verify_token_budget = self._verify_planner.resolve_verify_token_budget(
|
||||
draft_input=draft_input,
|
||||
@@ -939,7 +927,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
fold_eligible = (
|
||||
self._verify_executor.verify_epilogue is not None
|
||||
and proposal.folded
|
||||
# The epilogue's in-graph accept is greedy (accept_greedy_triton);
|
||||
# The epilogue's in-graph accept is greedy (AcceptGreedy);
|
||||
# sampling batches must take the eager accept path even when the
|
||||
# draft proposal itself folded.
|
||||
and (sampling_info is None or sampling_info.is_all_greedy)
|
||||
@@ -1075,6 +1063,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
target_logits=logits_output.next_token_logits,
|
||||
layout=layout,
|
||||
confidence=confidence,
|
||||
confidence_raw=proposal.confidence_raw,
|
||||
prefix_lens=prefix_lens,
|
||||
draft_tokens=draft_tokens,
|
||||
draft_block=draft_block,
|
||||
|
||||
Reference in New Issue
Block a user