[DSpark] Share draft params, logits dtype, confidence and the accept tail between graph and eager (#43790)

This commit is contained in:
Liangsheng Yin
2026-10-12 11:35:41 -07:00
committed by GitHub
parent f92199b569
commit 392039d752
11 changed files with 460 additions and 289 deletions
+9 -24
View File
@@ -65,16 +65,11 @@ from sglang.srt.speculative.ragged_verify import (
read_ragged_verify_mode,
)
from sglang.srt.utils import add_prefix, is_npu
from sglang.srt.utils.invariants import Bucket, InClosedRange, Invariant, expect
logger = logging.getLogger(__name__)
_PAD_NUM_HEADS = 64
# DSpark confidence is a per-token score that must stay in [0, 1].
_CONFIDENCE = Invariant(
"dspark.model.confidence", Bucket.GUARD, InClosedRange(0.0, 1.0)
)
_is_npu = is_npu()
@@ -1002,25 +997,15 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
self,
*,
anchor_tokens: torch.Tensor,
sampled_tokens: torch.Tensor,
x_post_hc: torch.Tensor,
) -> Optional[torch.Tensor]:
confidence_head = self.confidence_head
if confidence_head is None:
return None
bs = int(anchor_tokens.shape[0])
x_post_hc = x_post_hc.view(bs, self.gamma, -1)
if confidence_head.with_markov:
prev_seq = torch.cat(
[anchor_tokens.view(-1, 1), sampled_tokens[:, : self.gamma - 1]], dim=1
)
markov_embed_stack = self.markov_head.get_prev_embeddings(prev_seq)
else:
markov_embed_stack = None
confidence_raw = confidence_head(x_post_hc, markov_embed_stack)
confidence = confidence_head.apply_sts(confidence_raw)
expect(_CONFIDENCE, confidence)
return confidence
draft_tokens: torch.Tensor,
confidence_features: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
return self.confidence_head.predict(
confidence_features,
anchor_tokens=anchor_tokens,
draft_tokens=draft_tokens,
markov_head=self.markov_head,
)
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> None:
params_dict = dict(self.named_parameters())
+47 -5
View File
@@ -25,12 +25,18 @@ from sglang.srt.speculative.ragged_verify import (
RaggedVerifyMode,
read_ragged_verify_mode,
)
from sglang.srt.utils.invariants import Bucket, InClosedRange, Invariant, expect
if TYPE_CHECKING:
from sglang.srt.distributed.device_communicators.vocab_gather import VocabGather
logger = logging.getLogger(__name__)
# DSpark confidence is a per-token score that must stay in [0, 1].
_CONFIDENCE = Invariant(
"dspark.model.confidence", Bucket.GUARD, InClosedRange(0.0, 1.0)
)
def gather_and_crop_vocab(
local_logits: torch.Tensor, lm_head: nn.Module
@@ -299,7 +305,6 @@ class DSparkConfidenceHead(nn.Module):
self.register_buffer(
"sts_temperatures", torch.ones((), dtype=torch.float32), persistent=False
)
self._last_confidence_raw: Optional[torch.Tensor] = None
def forward(
self,
@@ -321,9 +326,30 @@ class DSparkConfidenceHead(nn.Module):
return self.proj(features).squeeze(-1)
def apply_sts(self, confidence_raw: torch.Tensor) -> torch.Tensor:
self._last_confidence_raw = confidence_raw
return torch.sigmoid(confidence_raw.float() / self.sts_temperatures)
def predict(
self,
features: torch.Tensor,
*,
anchor_tokens: torch.Tensor,
draft_tokens: torch.Tensor,
markov_head: BaseDSparkMarkovHead,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""The [bs, gamma] confidence of a drafted block and its raw logit."""
bs, gamma = draft_tokens.shape
if self.with_markov:
prev_tokens = torch.cat(
[anchor_tokens.view(-1, 1), draft_tokens[:, : gamma - 1]], dim=1
)
markov_embed_stack = markov_head.get_prev_embeddings(prev_tokens)
else:
markov_embed_stack = None
confidence_raw = self(features.view(bs, gamma, -1), markov_embed_stack)
confidence = self.apply_sts(confidence_raw)
expect(_CONFIDENCE, confidence)
return confidence, confidence_raw
def build_confidence_head(config) -> Optional[nn.Module]:
if read_ragged_verify_mode() is RaggedVerifyMode.STATIC:
@@ -399,8 +425,9 @@ class DSparkDraftMixin:
def compute_base_logits(
self, hidden: torch.Tensor
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
"""Project the draft's raw final hidden through the target lm_head.
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Project the draft's raw final hidden through the target lm_head;
also returns the features the confidence head reads.
muP targets (Inkling) train the draft against a FOLDED head (weights
pre-divided by logits_mup_width_multiplier) while serving attaches the
@@ -414,11 +441,26 @@ class DSparkDraftMixin:
"DSpark dense draft requires the target lm_head "
"(call attach_shared_modules first)."
)
confidence_features = hidden
if self.logits_mup_width_multiplier:
hidden = hidden / self.logits_mup_width_multiplier
local_logits = project_through_lm_head(hidden, self.lm_head)
base_logits = gather_and_crop_vocab(local_logits, self.lm_head)
return base_logits, None
return base_logits, confidence_features
def compute_confidence(
self,
*,
anchor_tokens: torch.Tensor,
draft_tokens: torch.Tensor,
confidence_features: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
return self.confidence_head.predict(
confidence_features,
anchor_tokens=anchor_tokens,
draft_tokens=draft_tokens,
markov_head=self.markov_head,
)
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
markov_weights = []
@@ -1,7 +1,8 @@
from __future__ import annotations
import functools
import logging
from typing import TYPE_CHECKING, Optional
from typing import TYPE_CHECKING, NamedTuple, Optional
import msgspec
import torch
@@ -72,9 +73,9 @@ class DraftForwardResult(msgspec.Struct, frozen=True):
class DraftProposal(msgspec.Struct, frozen=True):
draft_block_ids: torch.Tensor
draft_block: DraftBlockResult
draft_hidden: Optional[torch.Tensor]
# [bs, gamma]; None when the draft has no confidence head.
confidence: Optional[torch.Tensor] = None
confidence_tap: Optional[torch.Tensor] = None
confidence_raw: Optional[torch.Tensor] = None
folded: bool = False
@@ -118,6 +119,51 @@ def resolve_greedy_mask(
return (sampling_info.top_ks <= 1).view(-1)
class DraftParams(NamedTuple):
temperatures: torch.Tensor
greedy_mask: torch.Tensor
# The temperature each step samples at: 0 on argmax rows.
step_temperatures: torch.Tensor
@functools.cache
def _zero(device: torch.device) -> torch.Tensor:
return torch.zeros((), dtype=torch.float32, device=device)
def resolve_draft_params(
*,
sampling_info,
bs: int,
device: Optional[torch.device] = None,
out: Optional[DraftParams] = None,
) -> DraftParams:
"""Written into the ``[:bs]`` rows of ``out`` when given (the graph's
static buffers), else into fresh tensors on ``device``."""
if out is not None:
device = out.temperatures.device
else:
out = DraftParams(
temperatures=torch.empty(bs, dtype=torch.float32, device=device),
greedy_mask=torch.empty(bs, dtype=torch.bool, device=device),
step_temperatures=torch.empty(bs, dtype=torch.float32, device=device),
)
temperatures, greedy_mask, step_temperatures = (buf[:bs] for buf in out)
if sampling_info is None:
temperatures.fill_(1.0)
greedy_mask.fill_(True)
step_temperatures.zero_()
else:
torch.clamp(
sampling_info.temperatures.view(-1)[:bs].to(torch.float32),
min=1e-5,
out=temperatures,
)
torch.le(sampling_info.top_ks.view(-1)[:bs], 1, out=greedy_mask)
torch.where(greedy_mask, _zero(device), temperatures, out=step_temperatures)
return DraftParams(temperatures, greedy_mask, step_temperatures)
# XORed into the server seed so the draft noise differs from other samplers'.
_DRAFT_NOISE_DOMAIN = 0x2D2A9C317E44B1F3
@@ -234,14 +280,6 @@ def draft_markov_block(
return torch.stack(draft_tokens, dim=1), logits_out
def resolve_draft_temperatures(
*, bs: int, sampling_info, device: torch.device
) -> torch.Tensor:
if sampling_info is None:
return torch.ones(bs, dtype=torch.float32, device=device)
return sampling_info.temperatures.view(-1).to(torch.float32).clamp_min(1e-5)
def sample_draft_block(
*,
base_logits: torch.Tensor,
@@ -254,17 +292,15 @@ def sample_draft_block(
tp_sync: SpecTpSync,
greedy_draft: bool,
) -> DraftBlockResult:
bs = base_logits.shape[0]
greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device)
temperatures = resolve_draft_temperatures(
bs=bs, sampling_info=sampling_info, device=device
params = resolve_draft_params(
sampling_info=sampling_info, bs=base_logits.shape[0], device=device
)
draft_tokens, step_logits = draft_markov_block(
markov_head=markov_head,
base_logits=base_logits,
anchor_tokens=anchor_tokens,
draft_hidden=draft_hidden,
step_temperatures=temperatures.masked_fill(greedy_mask, 0.0),
step_temperatures=params.step_temperatures,
noise=noise,
tp_sync=tp_sync,
greedy_draft=greedy_draft,
@@ -274,8 +310,8 @@ def sample_draft_block(
corrected_logits=(
None if greedy_draft else expect(_DRAFT_STEP_LOGITS, step_logits)
),
greedy_mask=greedy_mask,
temperatures=temperatures,
greedy_mask=params.greedy_mask,
temperatures=params.temperatures,
greedy_draft=greedy_draft,
)
@@ -359,8 +395,7 @@ class DraftBlockProposer:
)
draft_block_ids = fwd.draft_block_ids
folded_confidence = None
confidence_tap = None
confidence = confidence_raw = None
folded = False
if (
envs.SGLANG_DSPARK_FOLDED_PROPOSAL.get()
@@ -373,15 +408,16 @@ class DraftBlockProposer:
corrected_logits=(
None if self._greedy_draft else draft_sampler.step_logits[:bs]
),
greedy_mask=draft_sampler.greedy_mask[:bs],
temperatures=draft_sampler.temperatures[:bs],
greedy_mask=draft_sampler.params.greedy_mask[:bs],
temperatures=draft_sampler.params.temperatures[:bs],
greedy_draft=self._greedy_draft,
)
if draft_sampler.confidence_out is not None:
folded_confidence = draft_sampler.confidence_out[:bs]
confidence = draft_sampler.confidence_out[:bs]
confidence_raw = draft_sampler.confidence_raw_out[:bs]
else:
with self._base_logits_context():
base_logits, confidence_tap = self.draft_model.compute_base_logits(
base_logits, confidence_features = self.draft_model.compute_base_logits(
fwd.raw_hidden
)
base_logits = base_logits.view(bs, self.gamma, -1)
@@ -396,6 +432,12 @@ class DraftBlockProposer:
tp_sync=self._tp_sync,
greedy_draft=self._greedy_draft,
)
if self.draft_model.confidence_head is not None:
confidence, confidence_raw = self.draft_model.compute_confidence(
anchor_tokens=draft_block_ids[:, 0],
draft_tokens=draft_block.draft_tokens,
confidence_features=confidence_features,
)
proposal_block_ids = (
draft_block_ids
if self.sample_from_anchor
@@ -404,9 +446,8 @@ class DraftBlockProposer:
return DraftProposal(
draft_block_ids=proposal_block_ids,
draft_block=draft_block,
draft_hidden=fwd.draft_hidden_3d,
confidence=folded_confidence,
confidence_tap=confidence_tap,
confidence=confidence,
confidence_raw=confidence_raw,
folded=folded,
)
@@ -7,7 +7,9 @@ import torch
from sglang.srt.speculative.dspark_components.dspark_draft import (
DraftNoise,
DraftParams,
draft_markov_block,
resolve_draft_params,
select_draft_hidden_without_anchor,
step_logits_width,
)
@@ -29,7 +31,6 @@ class DsparkDraftSampler:
noise: DraftNoise,
tp_sync: SpecTpSync,
greedy_draft: bool,
confidence_fn=None,
out=None,
):
self.model = model
@@ -46,55 +47,47 @@ class DsparkDraftSampler:
self.out = torch.empty(
(self.max_bs * self.gamma,), dtype=torch.int64, device=device
)
self.confidence_fn = confidence_fn
self.confidence_out = (
torch.empty((self.max_bs, self.gamma), dtype=torch.float32, device=device)
if confidence_fn is not None
else None
)
confidence_head = model.confidence_head
if confidence_head is None:
self.confidence_out = self.confidence_raw_out = None
else:
self.confidence_out = torch.empty(
(self.max_bs, self.gamma), dtype=torch.float32, device=device
)
self.confidence_raw_out = torch.empty(
(self.max_bs, self.gamma),
dtype=confidence_head.proj.weight.dtype,
device=device,
)
self.greedy_draft = greedy_draft
self._noise = noise
self._tp_sync = tp_sync
# Staged before each replay; step_temperatures is 0 on argmax rows.
self.temperatures = torch.ones(
(self.max_bs,), dtype=torch.float32, device=device
# Staged before each replay.
self.params = DraftParams(
temperatures=torch.ones((self.max_bs,), dtype=torch.float32, device=device),
greedy_mask=torch.ones((self.max_bs,), dtype=torch.bool, device=device),
step_temperatures=torch.zeros(
(self.max_bs,), dtype=torch.float32, device=device
),
)
self.greedy_mask = torch.ones((self.max_bs,), dtype=torch.bool, device=device)
self.step_temperatures = torch.zeros(
(self.max_bs,), dtype=torch.float32, device=device
)
self._zero = torch.zeros((), dtype=torch.float32, device=device)
# [max_bs, gamma, vocab] step logits for the sampling accept; a greedy
# draft keeps none.
self.step_logits = (
None
if greedy_draft
else torch.empty(
(self.max_bs, self.gamma, _step_logits_width(model)),
dtype=_step_logits_dtype(model),
device=device,
# [max_bs, gamma, vocab] step logits for the sampling accept, sized on
# the first call; a greedy draft keeps none.
self.step_logits: Optional[torch.Tensor] = None
def _step_logits(self, base_logits: torch.Tensor) -> torch.Tensor:
if self.step_logits is None:
# The runner's warmup calls the tail eagerly before capture.
assert not torch.cuda.is_current_stream_capturing()
self.step_logits = torch.empty(
(self.max_bs, self.gamma, _step_logits_width(self.model)),
dtype=self.markov_head.step_logits_dtype(base_logits.dtype),
device=base_logits.device,
)
)
return self.step_logits
def stage_sampling_params(self, *, bs: int, sampling_info) -> None:
"""Must run before the draft graph replay that reads these buffers."""
if sampling_info is None:
self.temperatures[:bs].fill_(1.0)
self.greedy_mask[:bs].fill_(True)
self.step_temperatures[:bs].zero_()
return
torch.clamp(
sampling_info.temperatures.view(-1)[:bs].to(torch.float32),
min=1e-5,
out=self.temperatures[:bs],
)
torch.le(sampling_info.top_ks.view(-1)[:bs], 1, out=self.greedy_mask[:bs])
torch.where(
self.greedy_mask[:bs],
self._zero,
self.temperatures[:bs],
out=self.step_temperatures[:bs],
)
resolve_draft_params(sampling_info=sampling_info, bs=bs, out=self.params)
def __call__(self, hidden_states, input_ids):
bs = hidden_states.shape[0] // self.query_token_num
@@ -107,7 +100,7 @@ class DsparkDraftSampler:
bs=bs,
gamma=self.gamma,
)
base_logits, confidence_tap = self.model.compute_base_logits(model_hidden)
base_logits, confidence_features = self.model.compute_base_logits(model_hidden)
base_logits = base_logits.view(bs, self.gamma, -1)
anchor = input_ids.view(bs, self.query_token_num)[:, 0]
@@ -116,21 +109,23 @@ class DsparkDraftSampler:
base_logits=base_logits,
anchor_tokens=anchor,
draft_hidden=sample_hidden,
step_temperatures=self.step_temperatures[:bs],
step_temperatures=self.params.step_temperatures[:bs],
noise=self._noise,
tp_sync=self._tp_sync,
logits_out=None if self.greedy_draft else self.step_logits[:bs],
logits_out=None
if self.greedy_draft
else self._step_logits(base_logits)[:bs],
greedy_draft=self.greedy_draft,
)
self.out[: draft_tokens.numel()].copy_(draft_tokens.reshape(-1))
if self.confidence_out is not None:
confidence = self.confidence_fn(
draft_hidden=sample_hidden,
confidence, confidence_raw = self.model.compute_confidence(
anchor_tokens=anchor,
draft_tokens=draft_tokens,
confidence_tap=confidence_tap,
confidence_features=confidence_features,
)
self.confidence_out[:bs].copy_(confidence)
self.confidence_raw_out[:bs].copy_(confidence_raw)
def _step_logits_width(model) -> int:
@@ -138,16 +133,6 @@ def _step_logits_width(model) -> int:
return step_logits_width(model.markov_head, int(model.lm_head.org_vocab_size))
def _step_logits_dtype(model) -> torch.dtype:
# A quantized head's packed weight has no logits dtype; it emits the draft's.
weight = model.lm_head.weight
if weight.is_floating_point():
base_dtype = weight.dtype
else:
base_dtype = next(model.markov_head.parameters()).dtype
return model.markov_head.step_logits_dtype(base_dtype)
def maybe_build_draft_sampler(
*,
draft_model,
@@ -158,7 +143,6 @@ def maybe_build_draft_sampler(
noise: DraftNoise,
greedy_draft: bool,
tp_sync: SpecTpSync,
confidence_fn=None,
out=None,
) -> Optional[DsparkDraftSampler]:
"""Build the graph-folded draft sampler, or None (reason logged) when the
@@ -186,6 +170,5 @@ def maybe_build_draft_sampler(
noise=noise,
tp_sync=tp_sync,
greedy_draft=greedy_draft,
confidence_fn=confidence_fn,
out=out,
)
@@ -824,6 +824,7 @@ class DsparkStepObservers:
target_logits: Optional[torch.Tensor],
layout,
confidence: Optional[torch.Tensor],
confidence_raw: Optional[torch.Tensor],
prefix_lens: torch.Tensor,
draft_tokens: torch.Tensor,
draft_block,
@@ -838,20 +839,20 @@ class DsparkStepObservers:
dp_tier_num_tokens: Optional[int],
) -> None:
planner = self._planner
if not proposal_folded:
self._maybe_record_sts_collect(
verify_ids_2d=verify_ids_2d,
target_logits=target_logits,
bs=bs,
)
self._confidence_probe.maybe_observe(
carries_confidence=planner.carries_confidence,
is_compact_mode=planner.is_compact_mode,
confidence_raw=planner.last_confidence_raw,
verify_ids_2d=verify_ids_2d,
target_logits=target_logits,
bs=bs,
)
self._maybe_record_sts_collect(
confidence_raw=confidence_raw,
verify_ids_2d=verify_ids_2d,
target_logits=target_logits,
bs=bs,
)
self._confidence_probe.maybe_observe(
carries_confidence=planner.carries_confidence,
is_compact_mode=planner.is_compact_mode,
confidence_raw=confidence_raw,
verify_ids_2d=verify_ids_2d,
target_logits=target_logits,
bs=bs,
)
if self._block_accept_recorder is not None and not proposal_folded:
self._block_accept_recorder.observe_verify_step(
forward_ct=forward_ct,
@@ -929,15 +930,13 @@ class DsparkStepObservers:
def _maybe_record_sts_collect(
self,
*,
confidence_raw: Optional[torch.Tensor],
verify_ids_2d: torch.Tensor,
target_logits: Optional[torch.Tensor],
bs: int,
) -> None:
if not self._sts_collect_path:
return
if not self._planner.carries_confidence:
return
confidence_raw = self._planner.last_confidence_raw
if confidence_raw is None:
return
if self._sts_recorder is None:
@@ -45,7 +45,6 @@ from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite
from sglang.srt.utils.common import require_mlp_tp_gather
from sglang.srt.utils.invariants import (
Bucket,
InClosedRange,
Invariant,
IsTrue,
expect,
@@ -54,10 +53,6 @@ from sglang.srt.utils.invariants import (
logger = logging.getLogger(__name__)
# DSpark confidence is a per-token score that must stay in [0, 1].
_CONFIDENCE = Invariant(
"dspark.planner.confidence", Bucket.GUARD, InClosedRange(0.0, 1.0)
)
# Scheduled verify lengths must not exceed the per-step token budget.
_VERIFY_LEN_BUDGET = Invariant("dspark.verify_len_budget", Bucket.GUARD, IsTrue())
@@ -214,12 +209,6 @@ class DSparkVerifyPlanner:
def carries_confidence(self) -> bool:
return self._confidence_head is not None
@property
def last_confidence_raw(self) -> Optional[torch.Tensor]:
if self._confidence_head is None:
return None
return self._confidence_head._last_confidence_raw
@property
def schedules_verify_budget(self) -> bool:
return self._budget_planner is not None
@@ -252,37 +241,6 @@ class DSparkVerifyPlanner:
self._ragged_verify_mode is RaggedVerifyMode.COMPACT and layout is not None
)
def compute_confidence_tensor(
self,
*,
draft_hidden: Optional[torch.Tensor],
anchor_tokens: torch.Tensor,
draft_tokens: torch.Tensor,
confidence_tap: Optional[torch.Tensor] = None,
) -> Optional[torch.Tensor]:
if self._confidence_head is None:
return None
compute_confidence_hook = getattr(self.draft_model, "compute_confidence", None)
if compute_confidence_hook is not None:
assert confidence_tap is not None, (
"dsv4 compute_confidence needs the compute_base_logits tap"
)
with torch.inference_mode():
return compute_confidence_hook(
anchor_tokens=anchor_tokens,
sampled_tokens=draft_tokens,
x_post_hc=confidence_tap,
)
assert draft_hidden is not None
return compute_confidence(
draft_hidden=draft_hidden,
anchor_tokens=anchor_tokens,
draft_tokens=draft_tokens,
confidence_head=self._confidence_head,
markov_head=self.draft_model.markov_head,
gamma=self.gamma,
)
def prepare_verify_budget(
self, batch: ScheduleBatch, future_map: FutureMap
) -> None:
@@ -877,44 +835,6 @@ def apply_logits_adjustments_strided(
)
def build_markov_embed_stack(
*,
anchor_tokens: torch.Tensor,
draft_tokens: torch.Tensor,
markov_head,
gamma: int,
) -> torch.Tensor:
prev_seq = torch.cat(
[anchor_tokens.view(-1, 1), draft_tokens[:, : gamma - 1]], dim=1
)
return markov_head.get_prev_embeddings(prev_seq)
def compute_confidence(
*,
draft_hidden: torch.Tensor,
anchor_tokens: torch.Tensor,
draft_tokens: torch.Tensor,
confidence_head,
markov_head,
gamma: int,
) -> torch.Tensor:
assert confidence_head is not None
if confidence_head.with_markov:
markov_embed_stack = build_markov_embed_stack(
anchor_tokens=anchor_tokens,
draft_tokens=draft_tokens,
markov_head=markov_head,
gamma=gamma,
)
else:
markov_embed_stack = None
confidence_raw = confidence_head(draft_hidden, markov_embed_stack)
confidence = confidence_head.apply_sts(confidence_raw)
expect(_CONFIDENCE, confidence)
return confidence
class DSparkScheduleConfig(msgspec.Struct):
gamma: int
min_verify_len: int = 1
@@ -14,8 +14,6 @@ from sglang.kernels.ops.speculative.dspark.dspark_accept import (
FinalizeAcceptLens,
SelectMixedAccept,
SoftmaxTemp,
accept_greedy_triton,
finalize_accept_lens_triton,
)
from sglang.kernels.ops.speculative.dspark.dspark_verify_window import (
BuildCommitInjectLayout,
@@ -183,46 +181,24 @@ class TargetVerifyExecutor:
fused_argmax=self._target_is_dsv41,
block_verification=self._block_verification,
)
if simulate:
correct_len = self._simulated_correct_len.repeat(bs).to(correct_len.dtype)
if simulate_bonus_sampling_info is not None:
bonus = sample_simulated_bonus(
target_logits=target_logits,
correct_len=correct_len,
greedy_bonus=bonus,
sampling_info=simulate_bonus_sampling_info,
bs=bs,
verify_num_draft_tokens=self.verify_num_draft_tokens,
)
site = (
sync_site = (
SpecTpSyncSite.DSPARK_ACCEPT_GREEDY
if accept_sampling_info is None or accept_sampling_info.is_all_greedy
else SpecTpSyncSite.DSPARK_ACCEPT_SAMPLE
)
self._tp_sync.sync(site, correct_len)
self._tp_sync.sync(site, bonus)
self._tp_sync.sync(site, cap_trim_lens)
finalized = FinalizeAcceptLens.execute(
return build_accept_outs(
correct_len=correct_len,
bonus=bonus,
cap_trim_lens=cap_trim_lens,
prefix_lens=prefix_lens,
)
out_tokens = BuildOutTokens.execute(
draft_tokens=draft_tokens,
correct_len=correct_len,
bonus=bonus,
verify_num_draft_tokens=self.verify_num_draft_tokens,
gamma=self.gamma,
)
return AcceptOuts(
correct_len=correct_len,
bonus=bonus,
cap_trim_lens=finalized.cap_trim_lens,
commit_lens=finalized.commit_lens,
new_seq_lens=finalized.new_seq_lens,
out_tokens=out_tokens,
verify_num_draft_tokens=self.verify_num_draft_tokens,
tp_sync=self._tp_sync,
sync_site=sync_site,
simulated_correct_len=self._simulated_correct_len,
target_logits=target_logits,
simulate_bonus_sampling_info=simulate_bonus_sampling_info,
)
def begin_simulated_step(self) -> None:
@@ -768,38 +744,27 @@ class DsparkVerifyEpilogue:
self, *, candidates, logits, draft_tokens, seq_lens, cutoff_verify_lens=None
) -> torch.Tensor:
bs = candidates.shape[0]
correct_len, bonus, cap_trim_lens = accept_greedy_triton(
correct_len, bonus, cap_trim_lens = AcceptGreedy.execute(
candidates=candidates,
target_logits=logits,
verify_num_draft_tokens=self.stride,
cutoff_verify_lens=cutoff_verify_lens,
fused_argmax=self._fused_argmax,
)
if self.simulated_correct_len is not None:
# The accept runs as served; only its length is replaced.
correct_len = self.simulated_correct_len.repeat(bs).to(correct_len.dtype)
self._tp_sync.sync(SpecTpSyncSite.DSPARK_ACCEPT_GRAPH, correct_len)
self._tp_sync.sync(SpecTpSyncSite.DSPARK_ACCEPT_GRAPH, bonus)
self._tp_sync.sync(SpecTpSyncSite.DSPARK_ACCEPT_GRAPH, cap_trim_lens)
finalized = finalize_accept_lens_triton(
correct_len=correct_len,
cap_trim_lens=cap_trim_lens,
prefix_lens=seq_lens[:bs],
)
out_tokens = BuildOutTokens.execute(
draft_tokens=draft_tokens,
accept = build_accept_outs(
correct_len=correct_len,
bonus=bonus,
verify_num_draft_tokens=self.stride,
cap_trim_lens=cap_trim_lens,
prefix_lens=seq_lens[:bs],
draft_tokens=draft_tokens,
gamma=self.gamma,
verify_num_draft_tokens=self.stride,
tp_sync=self._tp_sync,
sync_site=SpecTpSyncSite.DSPARK_ACCEPT_GRAPH,
simulated_correct_len=self.simulated_correct_len,
out=self.read_accept(bs),
)
self.correct_len_buf[:bs].copy_(correct_len)
self.bonus_buf[:bs].copy_(bonus)
self.cap_trim_lens_buf[:bs].copy_(cap_trim_lens.to(torch.int32))
self.commit_lens_buf[:bs].copy_(finalized.commit_lens)
self.new_seq_lens_buf[:bs].copy_(finalized.new_seq_lens)
self.out_tokens_buf[:bs].copy_(out_tokens.view(bs, self.stride))
return finalized.commit_lens
return accept.commit_lens
def _commit_inject(
self, commit_lens, verify_lens, seq_lens, req_pool_indices, bs: int
@@ -838,6 +803,72 @@ class DsparkVerifyEpilogue:
)
def build_accept_outs(
*,
correct_len: torch.Tensor,
bonus: torch.Tensor,
cap_trim_lens: torch.Tensor,
prefix_lens: torch.Tensor,
draft_tokens: torch.Tensor,
gamma: int,
verify_num_draft_tokens: int,
tp_sync: SpecTpSync,
sync_site: SpecTpSyncSite,
simulated_correct_len: Optional[torch.Tensor] = None,
target_logits: Optional[torch.Tensor] = None,
simulate_bonus_sampling_info=None,
out: Optional[AcceptOuts] = None,
) -> AcceptOuts:
"""``out``: static buffers (the in-graph accept's) to copy the results into."""
bs = correct_len.shape[0]
if simulated_correct_len is not None:
# The accept runs as served; only its length is replaced.
correct_len = simulated_correct_len.repeat(bs).to(correct_len.dtype)
if simulate_bonus_sampling_info is not None:
assert target_logits is not None
bonus = sample_simulated_bonus(
target_logits=target_logits,
correct_len=correct_len,
greedy_bonus=bonus,
sampling_info=simulate_bonus_sampling_info,
bs=bs,
verify_num_draft_tokens=verify_num_draft_tokens,
)
tp_sync.sync(sync_site, correct_len)
tp_sync.sync(sync_site, bonus)
tp_sync.sync(sync_site, cap_trim_lens)
finalized = FinalizeAcceptLens.execute(
correct_len=correct_len,
cap_trim_lens=cap_trim_lens,
prefix_lens=prefix_lens,
)
out_tokens = BuildOutTokens.execute(
draft_tokens=draft_tokens,
correct_len=correct_len,
bonus=bonus,
verify_num_draft_tokens=verify_num_draft_tokens,
gamma=gamma,
)
accept = AcceptOuts(
correct_len=correct_len,
bonus=bonus,
cap_trim_lens=finalized.cap_trim_lens,
commit_lens=finalized.commit_lens,
new_seq_lens=finalized.new_seq_lens,
out_tokens=out_tokens,
)
if out is not None:
out.correct_len.copy_(accept.correct_len)
out.bonus.copy_(accept.bonus)
out.cap_trim_lens.copy_(accept.cap_trim_lens)
out.commit_lens.copy_(accept.commit_lens)
out.new_seq_lens.copy_(accept.new_seq_lens)
out.out_tokens.copy_(accept.out_tokens)
return out
return accept
def sample_simulated_bonus(
*,
target_logits: torch.Tensor,
@@ -539,11 +539,6 @@ class DSparkWorkerV2(BaseSpecWorker):
noise=self._proposer.noise,
greedy_draft=self._greedy_draft,
tp_sync=self._tp_sync,
confidence_fn=(
self._verify_planner.compute_confidence_tensor
if self._verify_planner.carries_confidence
else None
),
out=(
self._verify_epilogue.draft_tokens_buf
if self._verify_epilogue is not None
@@ -899,13 +894,6 @@ class DSparkWorkerV2(BaseSpecWorker):
coordination_plan.apply(batch, "target", rank, local_only=target_local_only)
confidence = proposal.confidence
if confidence is None:
confidence = self._verify_planner.compute_confidence_tensor(
draft_hidden=proposal.draft_hidden,
anchor_tokens=draft_block_ids[:, 0],
draft_tokens=draft_tokens,
confidence_tap=proposal.confidence_tap,
)
verify_token_budget = self._verify_planner.resolve_verify_token_budget(
draft_input=draft_input,
@@ -939,7 +927,7 @@ class DSparkWorkerV2(BaseSpecWorker):
fold_eligible = (
self._verify_executor.verify_epilogue is not None
and proposal.folded
# The epilogue's in-graph accept is greedy (accept_greedy_triton);
# The epilogue's in-graph accept is greedy (AcceptGreedy);
# sampling batches must take the eager accept path even when the
# draft proposal itself folded.
and (sampling_info is None or sampling_info.is_all_greedy)
@@ -1075,6 +1063,7 @@ class DSparkWorkerV2(BaseSpecWorker):
target_logits=logits_output.next_token_logits,
layout=layout,
confidence=confidence,
confidence_raw=proposal.confidence_raw,
prefix_lens=prefix_lens,
draft_tokens=draft_tokens,
draft_block=draft_block,