[DSpark] Draft the eager markov block through MarkovSampleStep (#43622)
This commit is contained in:
@@ -542,7 +542,6 @@ class Envs:
|
||||
SGLANG_DSPARK_ENABLE_SPS_RECORD = EnvBool(False)
|
||||
SGLANG_DSPARK_FAST_KERNEL = EnvBool(True)
|
||||
SGLANG_DSPARK_FP32_LM_HEAD = EnvBool(False)
|
||||
SGLANG_DSPARK_FAST_SAMPLING = EnvBool(True)
|
||||
SGLANG_DSPARK_FOLDED_SAMPLING = EnvInt(DsparkFoldedSampling.AUTO)
|
||||
SGLANG_DSPARK_FOLDED_PROPOSAL = EnvBool(True)
|
||||
SGLANG_DSPARK_STACKED_CTX_KV = EnvBool(True)
|
||||
@@ -1988,6 +1987,9 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
|
||||
note="Strategy-based prefill context parallelism is now the only generic implementation."
|
||||
),
|
||||
"SGLANG_TRACE_QWEN35_FINAL_NORM": _DeprecatedEnv(),
|
||||
"SGLANG_DSPARK_FAST_SAMPLING": _DeprecatedEnv(
|
||||
note="DSpark draft sampling always uses Gumbel-max."
|
||||
),
|
||||
"SGLANG_SKIP_RUST_TESTS": _DeprecatedEnv(
|
||||
note="The Rust tests run in pr-test-rust-workspace.yml now."
|
||||
),
|
||||
|
||||
@@ -51,6 +51,7 @@ from sglang.srt.models.deepseek_v4 import (
|
||||
from sglang.srt.models.dspark import (
|
||||
BaseDSparkMarkovHead,
|
||||
DSparkConfidenceHead,
|
||||
VocabShard,
|
||||
gather_and_crop_vocab,
|
||||
project_through_lm_head,
|
||||
)
|
||||
@@ -473,6 +474,9 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead):
|
||||
num_embeddings_per_partition=per_partition,
|
||||
num_embeddings_padded=num_padded,
|
||||
)
|
||||
self.vocab_shard = VocabShard(
|
||||
start=self._tp_shard.org_vocab_start, gather=self._vocab_gather
|
||||
)
|
||||
|
||||
def get_prev_embeddings(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.markov_w1(token_ids.long())
|
||||
@@ -514,6 +518,11 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead):
|
||||
full = self._vocab_gather(step_local)
|
||||
return full[..., : self.vocab_size]
|
||||
|
||||
def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype:
|
||||
# The bias is FP32 or upcast into an FP32 add, sharded or not.
|
||||
del base_dtype
|
||||
return torch.float32
|
||||
|
||||
@property
|
||||
def supports_sharded_greedy(self) -> bool:
|
||||
return (
|
||||
|
||||
@@ -2,8 +2,9 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Callable, Iterable, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Callable, Iterable, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
@@ -28,6 +29,9 @@ from sglang.srt.speculative.ragged_verify import (
|
||||
read_ragged_verify_mode,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.distributed.device_communicators.vocab_gather import VocabGather
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
StepSampler = Callable[[torch.Tensor, int], torch.Tensor]
|
||||
@@ -85,10 +89,21 @@ def run_markov_block(
|
||||
)
|
||||
|
||||
|
||||
class VocabShard(msgspec.Struct, frozen=True):
|
||||
"""Where a rank's vocab slice starts, and the gather over its group."""
|
||||
|
||||
start: int
|
||||
gather: VocabGather
|
||||
|
||||
|
||||
class BaseDSparkMarkovHead(nn.Module, ABC):
|
||||
"""Corrects a draft block step by step: each step's base logits get a bias
|
||||
computed from the token drafted at the step before."""
|
||||
|
||||
vocab_size: int
|
||||
# The rank's slice of a TP-sharded vocab; None when the vocab is whole.
|
||||
vocab_shard: Optional[VocabShard] = None
|
||||
|
||||
def init_state(
|
||||
self, hidden_states: Optional[torch.Tensor]
|
||||
) -> Optional[torch.Tensor]:
|
||||
@@ -110,6 +125,10 @@ class BaseDSparkMarkovHead(nn.Module, ABC):
|
||||
) -> torch.Tensor:
|
||||
return base_logits + bias
|
||||
|
||||
@abstractmethod
|
||||
def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype:
|
||||
"""The dtype of ``base_logits + bias``."""
|
||||
|
||||
@property
|
||||
def supports_sharded_greedy(self) -> bool:
|
||||
return False
|
||||
@@ -151,6 +170,10 @@ class VanillaMarkov(BaseDSparkMarkovHead):
|
||||
def project_bias(self, latent_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.markov_w2(latent_states)
|
||||
|
||||
def step_logits_dtype(self, base_dtype: torch.dtype) -> torch.dtype:
|
||||
# The bias comes out in the embedding's dtype.
|
||||
return torch.promote_types(base_dtype, self.markov_w1.weight.dtype)
|
||||
|
||||
def step_bias(
|
||||
self,
|
||||
prev_tokens: torch.Tensor,
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_draft_model import (
|
||||
SampleStepTokens,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.dspark.markov_sample import MarkovSampleStep
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
@@ -30,24 +28,14 @@ from sglang.srt.speculative.spec_utils import draft_tp_context
|
||||
from sglang.srt.utils.common import is_pin_memory_available
|
||||
from sglang.srt.utils.invariants import Bucket, Invariant, NotNaN, expect
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.models.dspark import BaseDSparkMarkovHead
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _one_hot_token0(probs: torch.Tensor) -> torch.Tensor:
|
||||
degenerate = torch.isnan(probs[:, :1])
|
||||
one_hot = torch.zeros_like(probs)
|
||||
one_hot[:, 0] = 1.0
|
||||
return torch.where(degenerate, one_hot, probs)
|
||||
|
||||
|
||||
# Draft step logits: NaN is a bug, but -inf is legitimate (masking). The data
|
||||
# layer lives downstream (probs one-hot below, or the fast kernel's clamp).
|
||||
# Draft step logits: NaN is a bug, but -inf is legitimate (masking).
|
||||
_DRAFT_STEP_LOGITS = Invariant("dspark.draft.step_logits", Bucket.GUARD, NotNaN())
|
||||
# Draft sampling probs: SOFTEN (tolerate + count), matching the original
|
||||
# unconditional clamp; an all-NaN row would otherwise make multinomial raise.
|
||||
_DRAFT_PROBS = Invariant(
|
||||
"dspark.draft.probs", Bucket.SOFTEN, NotNaN(), recover=_one_hot_token0
|
||||
)
|
||||
|
||||
|
||||
def _make_num_token_non_padded(
|
||||
@@ -125,74 +113,148 @@ def resolve_greedy_mask(
|
||||
return (sampling_info.top_ks <= 1).view(-1)
|
||||
|
||||
|
||||
# XORed into the server seed so the draft noise differs from other samplers'.
|
||||
_DRAFT_NOISE_DOMAIN = 0x2D2A9C317E44B1F3
|
||||
|
||||
|
||||
def draft_noise_key(random_seed: int) -> int:
|
||||
return (int(random_seed) ^ _DRAFT_NOISE_DOMAIN) & 0x7FFFFFFFFFFFFFFF
|
||||
|
||||
|
||||
class DraftNoise(msgspec.Struct, frozen=True):
|
||||
"""Philox key of the draft noise, and a GPU counter each block advances."""
|
||||
|
||||
key: int
|
||||
offset: torch.Tensor
|
||||
|
||||
|
||||
def step_logits_width(markov_head: BaseDSparkMarkovHead, base_width: int) -> int:
|
||||
# Kept step logits are gathered whole over a sharded vocab.
|
||||
return base_width if markov_head.vocab_shard is None else markov_head.vocab_size
|
||||
|
||||
|
||||
def _draft_step(
|
||||
markov_head: BaseDSparkMarkovHead,
|
||||
prev_tokens: torch.Tensor,
|
||||
base_logits: torch.Tensor,
|
||||
*,
|
||||
hidden_states: Optional[torch.Tensor],
|
||||
state: Optional[torch.Tensor],
|
||||
temperatures: torch.Tensor,
|
||||
noise: DraftNoise,
|
||||
step: int,
|
||||
logits_out: Optional[torch.Tensor],
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
bias, state = markov_head.step_bias(
|
||||
prev_tokens, hidden_states=hidden_states, state=state
|
||||
)
|
||||
shard = markov_head.vocab_shard
|
||||
sharded = (
|
||||
{}
|
||||
if shard is None
|
||||
else dict(
|
||||
vocab_start=shard.start,
|
||||
gather=shard.gather.gather_stacked,
|
||||
gather_logits=shard.gather,
|
||||
)
|
||||
)
|
||||
tokens = MarkovSampleStep.execute(
|
||||
base_logits=base_logits,
|
||||
bias=bias,
|
||||
temperatures=temperatures,
|
||||
key=noise.key,
|
||||
offset=noise.offset,
|
||||
step=step,
|
||||
step_dtype=markov_head.step_logits_dtype(base_logits.dtype),
|
||||
logits_out=logits_out,
|
||||
# Rows sample at their temperatures only when the step logits are kept.
|
||||
has_sampled_rows=None if logits_out is not None else False,
|
||||
**sharded,
|
||||
)
|
||||
return tokens, state
|
||||
|
||||
|
||||
def draft_markov_block(
|
||||
*,
|
||||
markov_head,
|
||||
base_logits: torch.Tensor,
|
||||
anchor_tokens: torch.Tensor,
|
||||
draft_hidden: Optional[torch.Tensor],
|
||||
step_temperatures: torch.Tensor,
|
||||
noise: DraftNoise,
|
||||
tp_sync: SpecTpSync,
|
||||
site: SpecTpSyncSite,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Returns the drafted tokens and the [bs, gamma, vocab] step logits."""
|
||||
bs, gamma = base_logits.shape[:2]
|
||||
logits_out = torch.empty(
|
||||
(bs, gamma, step_logits_width(markov_head, base_logits.shape[-1])),
|
||||
dtype=markov_head.step_logits_dtype(base_logits.dtype),
|
||||
device=base_logits.device,
|
||||
)
|
||||
state = markov_head.init_state(draft_hidden)
|
||||
prev_tokens = anchor_tokens.long()
|
||||
draft_tokens = []
|
||||
for step in range(gamma):
|
||||
prev_tokens, state = _draft_step(
|
||||
markov_head,
|
||||
prev_tokens,
|
||||
base_logits[:, step],
|
||||
hidden_states=None if draft_hidden is None else draft_hidden[:, step],
|
||||
state=state,
|
||||
temperatures=step_temperatures,
|
||||
noise=noise,
|
||||
step=step,
|
||||
logits_out=logits_out[:, step],
|
||||
)
|
||||
prev_tokens = tp_sync.sync(site, prev_tokens)
|
||||
draft_tokens.append(prev_tokens)
|
||||
noise.offset.add_(1)
|
||||
return torch.stack(draft_tokens, dim=1), logits_out
|
||||
|
||||
|
||||
def resolve_draft_temperatures(
|
||||
*, bs: int, sampling_info, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
if sampling_info is None:
|
||||
return torch.ones(bs, dtype=torch.float32, device=device)
|
||||
return sampling_info.temperatures.view(-1).to(torch.float32).clamp_min(1e-5)
|
||||
|
||||
|
||||
def sample_draft_block(
|
||||
*,
|
||||
base_logits: torch.Tensor,
|
||||
anchor_tokens: torch.Tensor,
|
||||
draft_hidden: torch.Tensor,
|
||||
draft_hidden: Optional[torch.Tensor],
|
||||
sampling_info,
|
||||
markov_head,
|
||||
noise: DraftNoise,
|
||||
device: torch.device,
|
||||
tp_sync: SpecTpSync,
|
||||
) -> DraftBlockResult:
|
||||
bs = base_logits.shape[0]
|
||||
greedy_mask = resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device)
|
||||
any_sampling = sampling_info is not None and not sampling_info.is_all_greedy
|
||||
fast_sampling = envs.SGLANG_DSPARK_FAST_SAMPLING.get()
|
||||
|
||||
if sampling_info is None:
|
||||
temperatures = torch.ones(bs, dtype=torch.float32, device=device)
|
||||
else:
|
||||
temperatures = (
|
||||
sampling_info.temperatures.view(-1).to(torch.float32).clamp_min(1e-5)
|
||||
)
|
||||
|
||||
if not any_sampling:
|
||||
|
||||
def sampler(step_logits: torch.Tensor, step_idx: int) -> torch.Tensor:
|
||||
expect(_DRAFT_STEP_LOGITS, step_logits, msg=f"step {step_idx}")
|
||||
return tp_sync.sync(
|
||||
SpecTpSyncSite.DSPARK_DRAFT_GREEDY, torch.argmax(step_logits, dim=-1)
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
def sampler(step_logits: torch.Tensor, step_idx: int) -> torch.Tensor:
|
||||
expect(_DRAFT_STEP_LOGITS, step_logits, msg=f"step {step_idx}")
|
||||
if fast_sampling:
|
||||
exp_noise = torch.empty(
|
||||
step_logits.shape, dtype=torch.float32, device=step_logits.device
|
||||
).exponential_(1)
|
||||
return tp_sync.sync(
|
||||
SpecTpSyncSite.DSPARK_DRAFT_SAMPLE,
|
||||
SampleStepTokens.execute(
|
||||
step_logits=step_logits,
|
||||
temperatures=temperatures,
|
||||
greedy_mask=greedy_mask,
|
||||
exp_noise=exp_noise,
|
||||
),
|
||||
)
|
||||
else:
|
||||
probs = torch.softmax(
|
||||
step_logits.float() / temperatures[:, None], dim=-1
|
||||
)
|
||||
probs = expect(_DRAFT_PROBS, probs)
|
||||
argmax_tokens = torch.argmax(step_logits, dim=-1)
|
||||
sampled_tokens = torch.multinomial(probs, num_samples=1).squeeze(-1)
|
||||
return tp_sync.sync(
|
||||
SpecTpSyncSite.DSPARK_DRAFT_MULTINOMIAL,
|
||||
torch.where(greedy_mask, argmax_tokens, sampled_tokens),
|
||||
)
|
||||
|
||||
draft_tokens, corrected_logits = markov_head.sample_block(
|
||||
base_logits,
|
||||
first_prev_tokens=anchor_tokens,
|
||||
hidden_states=draft_hidden,
|
||||
sampler=sampler,
|
||||
temperatures = resolve_draft_temperatures(
|
||||
bs=bs, sampling_info=sampling_info, device=device
|
||||
)
|
||||
draft_tokens, step_logits = draft_markov_block(
|
||||
markov_head=markov_head,
|
||||
base_logits=base_logits,
|
||||
anchor_tokens=anchor_tokens,
|
||||
draft_hidden=draft_hidden,
|
||||
step_temperatures=temperatures.masked_fill(greedy_mask, 0.0),
|
||||
noise=noise,
|
||||
tp_sync=tp_sync,
|
||||
site=(
|
||||
SpecTpSyncSite.DSPARK_DRAFT_SAMPLE
|
||||
if any_sampling
|
||||
else SpecTpSyncSite.DSPARK_DRAFT_GREEDY
|
||||
),
|
||||
)
|
||||
return DraftBlockResult(
|
||||
draft_tokens=draft_tokens,
|
||||
corrected_logits=corrected_logits,
|
||||
corrected_logits=expect(_DRAFT_STEP_LOGITS, step_logits),
|
||||
greedy_mask=greedy_mask,
|
||||
temperatures=temperatures,
|
||||
)
|
||||
@@ -208,6 +270,7 @@ class DraftBlockProposer:
|
||||
mask_token_id: int,
|
||||
draft_block_spec_info,
|
||||
tp_sync: SpecTpSync,
|
||||
random_seed: int,
|
||||
dp_moe_sync: bool = False,
|
||||
) -> None:
|
||||
self.draft_model = draft_model
|
||||
@@ -218,6 +281,12 @@ class DraftBlockProposer:
|
||||
self._mask_token_id = mask_token_id
|
||||
self._draft_block_spec_info = draft_block_spec_info
|
||||
self._tp_sync = tp_sync
|
||||
self._noise = DraftNoise(
|
||||
key=draft_noise_key(random_seed),
|
||||
offset=torch.zeros(
|
||||
1, dtype=torch.int64, device=self.draft_model_runner.device
|
||||
),
|
||||
)
|
||||
self._draft_sampler = None
|
||||
self._dp_moe_sync = dp_moe_sync
|
||||
# Persistent (bs, gamma) mask-token buffer: only column 0 (the bonus
|
||||
@@ -322,6 +391,7 @@ class DraftBlockProposer:
|
||||
draft_hidden=fwd.draft_hidden_3d,
|
||||
sampling_info=sampling_info,
|
||||
markov_head=self.draft_model.markov_head,
|
||||
noise=self._noise,
|
||||
device=device,
|
||||
tp_sync=self._tp_sync,
|
||||
)
|
||||
|
||||
@@ -323,6 +323,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
mask_token_id=self._mask_token_id,
|
||||
draft_block_spec_info=self._draft_block_spec_info,
|
||||
tp_sync=self._tp_sync,
|
||||
random_seed=target_worker.random_seed,
|
||||
dp_moe_sync=self._draft_is_moe and get_parallel().attn_dp_enabled,
|
||||
)
|
||||
self._verify_epilogue = None
|
||||
|
||||
Reference in New Issue
Block a user