[DSpark] Draft the eager markov block through MarkovSampleStep (#43622)

This commit is contained in:
Liangsheng Yin
2026-10-11 15:13:02 -07:00
committed by GitHub
parent 467209e838
commit 48abbdb845
6 changed files with 261 additions and 73 deletions
+3 -1
View File
@@ -542,7 +542,6 @@ class Envs:
SGLANG_DSPARK_ENABLE_SPS_RECORD = EnvBool(False)
SGLANG_DSPARK_FAST_KERNEL = EnvBool(True)
SGLANG_DSPARK_FP32_LM_HEAD = EnvBool(False)
SGLANG_DSPARK_FAST_SAMPLING = EnvBool(True)
SGLANG_DSPARK_FOLDED_SAMPLING = EnvInt(DsparkFoldedSampling.AUTO)
SGLANG_DSPARK_FOLDED_PROPOSAL = EnvBool(True)
SGLANG_DSPARK_STACKED_CTX_KV = EnvBool(True)
@@ -1988,6 +1987,9 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
note="Strategy-based prefill context parallelism is now the only generic implementation."
),
"SGLANG_TRACE_QWEN35_FINAL_NORM": _DeprecatedEnv(),
"SGLANG_DSPARK_FAST_SAMPLING": _DeprecatedEnv(
note="DSpark draft sampling always uses Gumbel-max."
),
"SGLANG_SKIP_RUST_TESTS": _DeprecatedEnv(
note="The Rust tests run in pr-test-rust-workspace.yml now."
),
@@ -51,6 +51,7 @@ from sglang.srt.models.deepseek_v4 import (
from sglang.srt.models.dspark import (
BaseDSparkMarkovHead,
DSparkConfidenceHead,
VocabShard,
gather_and_crop_vocab,
project_through_lm_head,
)
@@ -473,6 +474,9 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead):
num_embeddings_per_partition=per_partition,
num_embeddings_padded=num_padded,
)
self.vocab_shard = VocabShard(
start=self._tp_shard.org_vocab_start, gather=self._vocab_gather
)
def get_prev_embeddings(self, token_ids: torch.Tensor) -> torch.Tensor:
return self.markov_w1(token_ids.long())
@@ -514,6 +518,11 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead):
full = self._vocab_gather(step_local)
return full[..., : self.vocab_size]
def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype:
# The bias is FP32 or upcast into an FP32 add, sharded or not.
del base_dtype
return torch.float32
@property
def supports_sharded_greedy(self) -> bool:
return (
+24 -1
View File
@@ -2,8 +2,9 @@ from __future__ import annotations
import logging
from abc import ABC, abstractmethod
from typing import Callable, Iterable, Optional, Tuple
from typing import TYPE_CHECKING, Callable, Iterable, Optional, Tuple
import msgspec
import torch
import torch.nn.functional as F
from torch import nn
@@ -28,6 +29,9 @@ from sglang.srt.speculative.ragged_verify import (
read_ragged_verify_mode,
)
if TYPE_CHECKING:
from sglang.srt.distributed.device_communicators.vocab_gather import VocabGather
logger = logging.getLogger(__name__)
StepSampler = Callable[[torch.Tensor, int], torch.Tensor]
@@ -85,10 +89,21 @@ def run_markov_block(
)
class VocabShard(msgspec.Struct, frozen=True):
"""Where a rank's vocab slice starts, and the gather over its group."""
start: int
gather: VocabGather
class BaseDSparkMarkovHead(nn.Module, ABC):
"""Corrects a draft block step by step: each step's base logits get a bias
computed from the token drafted at the step before."""
vocab_size: int
# The rank's slice of a TP-sharded vocab; None when the vocab is whole.
vocab_shard: Optional[VocabShard] = None
def init_state(
self, hidden_states: Optional[torch.Tensor]
) -> Optional[torch.Tensor]:
@@ -110,6 +125,10 @@ class BaseDSparkMarkovHead(nn.Module, ABC):
) -> torch.Tensor:
return base_logits + bias
@abstractmethod
def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype:
"""The dtype of ``base_logits + bias``."""
@property
def supports_sharded_greedy(self) -> bool:
return False
@@ -151,6 +170,10 @@ class VanillaMarkov(BaseDSparkMarkovHead):
def project_bias(self, latent_states: torch.Tensor) -> torch.Tensor:
return self.markov_w2(latent_states)
def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype:
# The bias comes out in the embedding's dtype.
return torch.promote_types(base_dtype, self.markov_w1.weight.dtype)
def step_bias(
self,
prev_tokens: torch.Tensor,
@@ -1,14 +1,12 @@
from __future__ import annotations
import logging
from typing import Optional
from typing import TYPE_CHECKING, Optional
import msgspec
import torch
from sglang.kernels.ops.speculative.dspark.dspark_draft_model import (
SampleStepTokens,
)
from sglang.kernels.ops.speculative.dspark.markov_sample import MarkovSampleStep
from sglang.srt.environ import envs
from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.managers.schedule_batch import ScheduleBatch
@@ -30,24 +28,14 @@ from sglang.srt.speculative.spec_utils import draft_tp_context
from sglang.srt.utils.common import is_pin_memory_available
from sglang.srt.utils.invariants import Bucket, Invariant, NotNaN, expect
if TYPE_CHECKING:
from sglang.srt.models.dspark import BaseDSparkMarkovHead
logger = logging.getLogger(__name__)
def _one_hot_token0(probs: torch.Tensor) -> torch.Tensor:
degenerate = torch.isnan(probs[:, :1])
one_hot = torch.zeros_like(probs)
one_hot[:, 0] = 1.0
return torch.where(degenerate, one_hot, probs)
# Draft step logits: NaN is a bug, but -inf is legitimate (masking). The data
# layer lives downstream (probs one-hot below, or the fast kernel's clamp).
# Draft step logits: NaN is a bug, but -inf is legitimate (masking).
_DRAFT_STEP_LOGITS = Invariant("dspark.draft.step_logits", Bucket.GUARD, NotNaN())
# Draft sampling probs: SOFTEN (tolerate + count), matching the original
# unconditional clamp; an all-NaN row would otherwise make multinomial raise.
_DRAFT_PROBS = Invariant(
"dspark.draft.probs", Bucket.SOFTEN, NotNaN(), recover=_one_hot_token0
)
def _make_num_token_non_padded(
@@ -125,74 +113,148 @@ def resolve_greedy_mask(
return (sampling_info.top_ks <= 1).view(-1)
# XORed into the server seed so the draft noise differs from other samplers'.
_DRAFT_NOISE_DOMAIN = 0x2D2A9C317E44B1F3
def draft_noise_key(random_seed: int) -> int:
return (int(random_seed) ^ _DRAFT_NOISE_DOMAIN) & 0x7FFFFFFFFFFFFFFF
class DraftNoise(msgspec.Struct, frozen=True):
"""Philox key of the draft noise, and a GPU counter each block advances."""
key: int
offset: torch.Tensor
def step_logits_width(markov_head: BaseDSparkMarkovHead, base_width: int) -> int:
# Kept step logits are gathered whole over a sharded vocab.
return base_width if markov_head.vocab_shard is None else markov_head.vocab_size
def _draft_step(
markov_head: BaseDSparkMarkovHead,
prev_tokens: torch.Tensor,
base_logits: torch.Tensor,
*,
hidden_states: Optional[torch.Tensor],
state: Optional[torch.Tensor],
temperatures: torch.Tensor,
noise: DraftNoise,
step: int,
logits_out: Optional[torch.Tensor],
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
bias, state = markov_head.step_bias(
prev_tokens, hidden_states=hidden_states, state=state
)
shard = markov_head.vocab_shard
sharded = (
{}
if shard is None
else dict(
vocab_start=shard.start,
gather=shard.gather.gather_stacked,
gather_logits=shard.gather,
)
)
tokens = MarkovSampleStep.execute(
base_logits=base_logits,
bias=bias,
temperatures=temperatures,
key=noise.key,
offset=noise.offset,
step=step,
step_dtype=markov_head.step_logits_dtype(base_logits.dtype),
logits_out=logits_out,
# Rows sample at their temperatures only when the step logits are kept.
has_sampled_rows=None if logits_out is not None else False,
**sharded,
)
return tokens, state
def draft_markov_block(
*,
markov_head,
base_logits: torch.Tensor,
anchor_tokens: torch.Tensor,
draft_hidden: Optional[torch.Tensor],
step_temperatures: torch.Tensor,
noise: DraftNoise,
tp_sync: SpecTpSync,
site: SpecTpSyncSite,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Returns the drafted tokens and the [bs, gamma, vocab] step logits."""
bs, gamma = base_logits.shape[:2]
logits_out = torch.empty(
(bs, gamma, step_logits_width(markov_head, base_logits.shape[-1])),
dtype=markov_head.step_logits_dtype(base_logits.dtype),
device=base_logits.device,
)
state = markov_head.init_state(draft_hidden)
prev_tokens = anchor_tokens.long()
draft_tokens = []
for step in range(gamma):
prev_tokens, state = _draft_step(
markov_head,
prev_tokens,
base_logits[:, step],
hidden_states=None if draft_hidden is None else draft_hidden[:, step],
state=state,
temperatures=step_temperatures,
noise=noise,
step=step,
logits_out=logits_out[:, step],
)
prev_tokens = tp_sync.sync(site, prev_tokens)
draft_tokens.append(prev_tokens)
noise.offset.add_(1)
return torch.stack(draft_tokens, dim=1), logits_out
def resolve_draft_temperatures(
*, bs: int, sampling_info, device: torch.device
) -> torch.Tensor:
if sampling_info is None:
return torch.ones(bs, dtype=torch.float32, device=device)
return sampling_info.temperatures.view(-1).to(torch.float32).clamp_min(1e-5)
def sample_draft_block(
*,
base_logits: torch.Tensor,
anchor_tokens: torch.Tensor,
draft_hidden: torch.Tensor,
draft_hidden: Optional[torch.Tensor],
sampling_info,
markov_head,
noise: DraftNoise,
device: torch.device,
tp_sync: SpecTpSync,
) -> DraftBlockResult:
bs = base_logits.shape[0]
greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device)
any_sampling = sampling_info is not None and not sampling_info.is_all_greedy
fast_sampling = envs.SGLANG_DSPARK_FAST_SAMPLING.get()
if sampling_info is None:
temperatures = torch.ones(bs, dtype=torch.float32, device=device)
else:
temperatures = (
sampling_info.temperatures.view(-1).to(torch.float32).clamp_min(1e-5)
)
if not any_sampling:
def sampler(step_logits: torch.Tensor, step_idx: int) -> torch.Tensor:
expect(_DRAFT_STEP_LOGITS, step_logits, msg=f"step {step_idx}")
return tp_sync.sync(
SpecTpSyncSite.DSPARK_DRAFT_GREEDY, torch.argmax(step_logits, dim=-1)
)
else:
def sampler(step_logits: torch.Tensor, step_idx: int) -> torch.Tensor:
expect(_DRAFT_STEP_LOGITS, step_logits, msg=f"step {step_idx}")
if fast_sampling:
exp_noise = torch.empty(
step_logits.shape, dtype=torch.float32, device=step_logits.device
).exponential_(1)
return tp_sync.sync(
SpecTpSyncSite.DSPARK_DRAFT_SAMPLE,
SampleStepTokens.execute(
step_logits=step_logits,
temperatures=temperatures,
greedy_mask=greedy_mask,
exp_noise=exp_noise,
),
)
else:
probs = torch.softmax(
step_logits.float() / temperatures[:, None], dim=-1
)
probs = expect(_DRAFT_PROBS, probs)
argmax_tokens = torch.argmax(step_logits, dim=-1)
sampled_tokens = torch.multinomial(probs, num_samples=1).squeeze(-1)
return tp_sync.sync(
SpecTpSyncSite.DSPARK_DRAFT_MULTINOMIAL,
torch.where(greedy_mask, argmax_tokens, sampled_tokens),
)
draft_tokens, corrected_logits = markov_head.sample_block(
base_logits,
first_prev_tokens=anchor_tokens,
hidden_states=draft_hidden,
sampler=sampler,
temperatures = resolve_draft_temperatures(
bs=bs, sampling_info=sampling_info, device=device
)
draft_tokens, step_logits = draft_markov_block(
markov_head=markov_head,
base_logits=base_logits,
anchor_tokens=anchor_tokens,
draft_hidden=draft_hidden,
step_temperatures=temperatures.masked_fill(greedy_mask, 0.0),
noise=noise,
tp_sync=tp_sync,
site=(
SpecTpSyncSite.DSPARK_DRAFT_SAMPLE
if any_sampling
else SpecTpSyncSite.DSPARK_DRAFT_GREEDY
),
)
return DraftBlockResult(
draft_tokens=draft_tokens,
corrected_logits=corrected_logits,
corrected_logits=expect(_DRAFT_STEP_LOGITS, step_logits),
greedy_mask=greedy_mask,
temperatures=temperatures,
)
@@ -208,6 +270,7 @@ class DraftBlockProposer:
mask_token_id: int,
draft_block_spec_info,
tp_sync: SpecTpSync,
random_seed: int,
dp_moe_sync: bool = False,
) -> None:
self.draft_model = draft_model
@@ -218,6 +281,12 @@ class DraftBlockProposer:
self._mask_token_id = mask_token_id
self._draft_block_spec_info = draft_block_spec_info
self._tp_sync = tp_sync
self._noise = DraftNoise(
key=draft_noise_key(random_seed),
offset=torch.zeros(
1, dtype=torch.int64, device=self.draft_model_runner.device
),
)
self._draft_sampler = None
self._dp_moe_sync = dp_moe_sync
# Persistent (bs, gamma) mask-token buffer: only column 0 (the bonus
@@ -322,6 +391,7 @@ class DraftBlockProposer:
draft_hidden=fwd.draft_hidden_3d,
sampling_info=sampling_info,
markov_head=self.draft_model.markov_head,
noise=self._noise,
device=device,
tp_sync=self._tp_sync,
)
@@ -323,6 +323,7 @@ class DSparkWorkerV2(BaseSpecWorker):
mask_token_id=self._mask_token_id,
draft_block_spec_info=self._draft_block_spec_info,
tp_sync=self._tp_sync,
random_seed=target_worker.random_seed,
dp_moe_sync=self._draft_is_moe and get_parallel().attn_dp_enabled,
)
self._verify_epilogue = None