From 9943cf8eeffc625bb50df88a913d97cab901190e Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sun, 11 Oct 2026 15:07:51 -0700 Subject: [PATCH] [DSpark] Run every markov head through one step interface (#43620) --- .../sglang/srt/models/deepseek_v4_dspark.py | 72 ++----- python/sglang/srt/models/dspark.py | 197 +++++++----------- .../dspark_components/dspark_draft_sampler.py | 2 +- 3 files changed, 97 insertions(+), 174 deletions(-) diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index 69c31855f0..a659dc75ad 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -48,11 +48,10 @@ from sglang.srt.models.deepseek_v4 import ( make_hc_head_params, ) from sglang.srt.models.dspark import ( + BaseDSparkMarkovHead, DSparkConfidenceHead, - StepSampler, gather_and_crop_vocab, project_through_lm_head, - run_markov_block, ) from sglang.srt.runtime_context import ( get_parallel, @@ -398,9 +397,7 @@ class MarkovW2ShardGeometry(msgspec.Struct, frozen=True): num_embeddings_padded: int -class DSparkV4MarkovHead(nn.Module): - markov_head_type = "vanilla" - +class DSparkV4MarkovHead(BaseDSparkMarkovHead): def __init__( self, *, vocab_size: int, markov_rank: int, is_dsv41: bool = False ) -> None: @@ -487,38 +484,32 @@ class DSparkV4MarkovHead(nn.Module): return F.linear(latent_states.to(weight.dtype), weight).float() return F.linear(latent_states.float(), weight) - def compute_step_bias( - self, token_ids: torch.Tensor, hidden_states: Optional[torch.Tensor] - ) -> torch.Tensor: - del hidden_states - return self.project_bias(self.get_prev_embeddings(token_ids)) - - def apply_step_logits( + def step_bias( self, - logits: torch.Tensor, + prev_tokens: torch.Tensor, *, - token_ids: torch.Tensor, hidden_states: Optional[torch.Tensor], - ) -> torch.Tensor: - if self._tp_shard is not None: - return self._apply_step_logits_sharded( - base_local=logits, token_ids=token_ids - ) - return logits + self.compute_step_bias(token_ids, hidden_states) - - def _apply_step_logits_sharded( - self, *, base_local: torch.Tensor, token_ids: torch.Tensor - ) -> torch.Tensor: + state: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + del hidden_states + latent = self.get_prev_embeddings(prev_tokens) shard = self._tp_shard - latent = self.get_prev_embeddings(token_ids) + if shard is None: + return self.project_bias(latent), state weight_local = self.markov_w2.weight[ shard.org_vocab_start : shard.org_vocab_end ] if self._opt_markov_w2_bf16: - bias = F.linear(latent.to(weight_local.dtype), weight_local) - else: - bias = F.linear(latent.float(), weight_local) - step_local = BuildStepLocal.execute(bias=bias, base_local=base_local) + # Keep the bias in BF16; the step add upcasts it to FP32. + return F.linear(latent.to(weight_local.dtype), weight_local), state + return F.linear(latent.float(), weight_local), state + + def step_logits( + self, base_logits: torch.Tensor, bias: torch.Tensor + ) -> torch.Tensor: + if self._tp_shard is None: + return base_logits + bias + step_local = BuildStepLocal.execute(bias=bias, base_local=base_logits) full = self._vocab_gather(step_local) return full[..., : self.vocab_size] @@ -536,13 +527,10 @@ class DSparkV4MarkovHead(nn.Module): ) shard = self._tp_shard - weight = self.markov_w2.weight[shard.org_vocab_start : shard.org_vocab_end] prev = first_prev_tokens.long() tokens = [] for step in range(base_logits.shape[1]): - latent = self.get_prev_embeddings(prev) - # Preserve the same BF16 GEMM rounding before the FP32 logits add. - bias = F.linear(latent.to(weight.dtype), weight) + bias, _ = self.step_bias(prev, hidden_states=None, state=None) prev = sharded_greedy_step( bias, base_logits[:, step], @@ -558,24 +546,6 @@ class DSparkV4MarkovHead(nn.Module): logits = self.project_bias(embed) return logits, embed - def sample_block( - self, - base_logits: torch.Tensor, - *, - first_prev_tokens: torch.Tensor, - hidden_states: Optional[torch.Tensor], - sampler: StepSampler, - collect_corrected: bool = True, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - return run_markov_block( - self, - base_logits, - first_prev_tokens=first_prev_tokens, - hidden_states=hidden_states, - sampler=sampler, - collect_corrected=collect_corrected, - ) - def build_dspark_v4_confidence_head( *, config: DeepSeekV4Config, markov_rank: int diff --git a/python/sglang/srt/models/dspark.py b/python/sglang/srt/models/dspark.py index c3640ea9c6..a4734c3a53 100644 --- a/python/sglang/srt/models/dspark.py +++ b/python/sglang/srt/models/dspark.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from abc import ABC, abstractmethod from typing import Callable, Iterable, Optional, Tuple import torch @@ -50,7 +51,7 @@ def project_through_lm_head(hidden: torch.Tensor, lm_head: nn.Module) -> torch.T def run_markov_block( - head: nn.Module, + head: BaseDSparkMarkovHead, base_logits: torch.Tensor, *, first_prev_tokens: torch.Tensor, @@ -58,6 +59,7 @@ def run_markov_block( sampler: StepSampler, collect_corrected: bool = True, ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + state = head.init_state(hidden_states) batch_size, proposal_len = base_logits.shape[:2] if proposal_len == 0: empty = torch.empty(batch_size, 0, dtype=torch.long, device=base_logits.device) @@ -68,11 +70,10 @@ def run_markov_block( prev_tokens = first_prev_tokens.long() for step_idx in range(proposal_len): step_hidden = None if hidden_states is None else hidden_states[:, step_idx, ...] - step_logits = head.apply_step_logits( - base_logits[:, step_idx, :], - token_ids=prev_tokens, - hidden_states=step_hidden, + bias, state = head.step_bias( + prev_tokens, hidden_states=step_hidden, state=state ) + step_logits = head.step_logits(base_logits[:, step_idx, :], bias) next_tokens = sampler(step_logits, step_idx) sampled_tokens.append(next_tokens) if collect_corrected: @@ -84,53 +85,34 @@ def run_markov_block( ) -class VanillaMarkov(nn.Module): - markov_head_type = "vanilla" +class BaseDSparkMarkovHead(nn.Module, ABC): + """Corrects a draft block step by step: each step's base logits get a bias + computed from the token drafted at the step before.""" - def __init__(self, *, vocab_size: int, markov_rank: int) -> None: - super().__init__() - self.vocab_size = int(vocab_size) - self.markov_rank = int(markov_rank) - if self.markov_rank <= 0: - raise ValueError( - f"VanillaMarkov requires markov_rank > 0, got {self.markov_rank}." - ) - self.markov_w1 = nn.Embedding(self.vocab_size, self.markov_rank) - self.markov_w2 = nn.Linear(self.markov_rank, self.vocab_size, bias=False) - - def get_prev_embeddings(self, token_ids: torch.Tensor) -> torch.Tensor: - return self.markov_w1(token_ids.long()) - - def project_bias(self, latent_states: torch.Tensor) -> torch.Tensor: - return self.markov_w2(latent_states) - - def compute_step_bias( - self, - token_ids: torch.Tensor, - hidden_states: Optional[torch.Tensor], - ) -> torch.Tensor: + def init_state( + self, hidden_states: Optional[torch.Tensor] + ) -> Optional[torch.Tensor]: del hidden_states - return self.project_bias(self.get_prev_embeddings(token_ids)) + return None - def apply_step_logits( + @abstractmethod + def step_bias( self, - logits: torch.Tensor, + prev_tokens: torch.Tensor, *, - token_ids: torch.Tensor, hidden_states: Optional[torch.Tensor], - ) -> torch.Tensor: - return logits + self.compute_step_bias(token_ids, hidden_states) + state: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Return this step's bias and the state the next step reads.""" - def apply_block_logits( - self, - base_logits: torch.Tensor, - *, - token_ids: torch.Tensor, - hidden_states: Optional[torch.Tensor], + def step_logits( + self, base_logits: torch.Tensor, bias: torch.Tensor ) -> torch.Tensor: - if base_logits.size(-2) == 0: - return base_logits - return base_logits + self.compute_step_bias(token_ids, hidden_states) + return base_logits + bias + + @property + def supports_sharded_greedy(self) -> bool: + return False def sample_block( self, @@ -150,6 +132,35 @@ class VanillaMarkov(nn.Module): collect_corrected=collect_corrected, ) + +class VanillaMarkov(BaseDSparkMarkovHead): + def __init__(self, *, vocab_size: int, markov_rank: int) -> None: + super().__init__() + self.vocab_size = int(vocab_size) + self.markov_rank = int(markov_rank) + if self.markov_rank <= 0: + raise ValueError( + f"VanillaMarkov requires markov_rank > 0, got {self.markov_rank}." + ) + self.markov_w1 = nn.Embedding(self.vocab_size, self.markov_rank) + self.markov_w2 = nn.Linear(self.markov_rank, self.vocab_size, bias=False) + + def get_prev_embeddings(self, token_ids: torch.Tensor) -> torch.Tensor: + return self.markov_w1(token_ids.long()) + + def project_bias(self, latent_states: torch.Tensor) -> torch.Tensor: + return self.markov_w2(latent_states) + + def step_bias( + self, + prev_tokens: torch.Tensor, + *, + hidden_states: Optional[torch.Tensor], + state: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + del hidden_states + return self.project_bias(self.get_prev_embeddings(prev_tokens)), state + def sample_block_greedy_fused( self, base_logits: torch.Tensor, @@ -219,8 +230,6 @@ class Nemotron35VanillaMarkov(VanillaMarkov): class GatedMarkovHead(VanillaMarkov): - markov_head_type = "gated" - def __init__(self, *, vocab_size: int, markov_rank: int, hidden_size: int) -> None: super().__init__(vocab_size=vocab_size, markov_rank=markov_rank) self.gate_proj = nn.Linear(int(hidden_size) + markov_rank, markov_rank) @@ -236,16 +245,18 @@ class GatedMarkovHead(VanillaMarkov): gate_inputs = torch.cat([hidden_states, prev_embeddings], dim=-1) return torch.sigmoid(self.gate_proj(gate_inputs)) - def compute_step_bias( + def step_bias( self, - token_ids: torch.Tensor, + prev_tokens: torch.Tensor, + *, hidden_states: Optional[torch.Tensor], - ) -> torch.Tensor: - prev_embeddings = self.get_prev_embeddings(token_ids) - gate = self.compute_gate(token_ids, hidden_states).to( + state: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + prev_embeddings = self.get_prev_embeddings(prev_tokens) + gate = self.compute_gate(prev_tokens, hidden_states).to( dtype=prev_embeddings.dtype ) - return self.project_bias(gate * prev_embeddings) + return self.project_bias(gate * prev_embeddings), state def sample_block_greedy_fused( self, @@ -259,8 +270,6 @@ class GatedMarkovHead(VanillaMarkov): class RNNHead(VanillaMarkov): - markov_head_type = "rnn" - def __init__(self, *, vocab_size: int, markov_rank: int, hidden_size: int) -> None: super().__init__(vocab_size=vocab_size, markov_rank=markov_rank) self.hidden_size = int(hidden_size) @@ -281,84 +290,28 @@ class RNNHead(VanillaMarkov): bias = self.project_bias(torch.tanh(output_raw)) return new_state, bias - def compute_step_bias( - self, - token_ids: torch.Tensor, - hidden_states: Optional[torch.Tensor], - ) -> torch.Tensor: + def init_state( + self, hidden_states: Optional[torch.Tensor] + ) -> Optional[torch.Tensor]: if hidden_states is None: raise ValueError("RNNHead requires hidden_states.") - prev_embeddings = self.get_prev_embeddings(token_ids) - state = torch.zeros_like(prev_embeddings) - _, bias = self._rnn_step(state, prev_embeddings, hidden_states) - return bias - - def apply_block_logits( - self, - base_logits: torch.Tensor, - *, - token_ids: torch.Tensor, - hidden_states: Optional[torch.Tensor], - ) -> torch.Tensor: - if hidden_states is None: - raise ValueError("RNNHead requires hidden_states.") - block_size = base_logits.size(-2) - if block_size == 0: - return base_logits - leading_shape = base_logits.shape[:-2] - state = torch.zeros( - *leading_shape, + return torch.zeros( + hidden_states.shape[0], self.markov_rank, - device=base_logits.device, + device=hidden_states.device, dtype=hidden_states.dtype, ) - output_logits = [] - for k in range(block_size): - prev_emb = self.get_prev_embeddings(token_ids[..., k]) - state, bias = self._rnn_step(state, prev_emb, hidden_states[..., k, :]) - output_logits.append(base_logits[..., k, :] + bias) - return torch.stack(output_logits, dim=-2) - def sample_block( + def step_bias( self, - base_logits: torch.Tensor, + prev_tokens: torch.Tensor, *, - first_prev_tokens: torch.Tensor, hidden_states: Optional[torch.Tensor], - sampler: StepSampler, - collect_corrected: bool = True, + state: Optional[torch.Tensor], ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - if hidden_states is None: - raise ValueError("RNNHead requires hidden_states.") - batch_size, proposal_len = base_logits.shape[:2] - if proposal_len == 0: - empty = torch.empty( - batch_size, 0, dtype=torch.long, device=base_logits.device - ) - return empty, base_logits - - state = torch.zeros( - batch_size, - self.markov_rank, - device=base_logits.device, - dtype=hidden_states.dtype, - ) - sampled_tokens = [] - corrected_logits = [] - prev_tokens = first_prev_tokens.long() - for step_idx in range(proposal_len): - prev_emb = self.get_prev_embeddings(prev_tokens) - state, bias = self._rnn_step(state, prev_emb, hidden_states[:, step_idx, :]) - step_logits = base_logits[:, step_idx, :] + bias - next_tokens = sampler(step_logits, step_idx) - sampled_tokens.append(next_tokens) - if collect_corrected: - corrected_logits.append(step_logits.unsqueeze(1)) - prev_tokens = next_tokens - return ( - torch.stack(sampled_tokens, dim=1), - torch.cat(corrected_logits, dim=1) if collect_corrected else None, - ) + prev_emb = self.get_prev_embeddings(prev_tokens) + state, bias = self._rnn_step(state, prev_emb, hidden_states) + return bias, state def sample_block_greedy_fused( self, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py index 95856e721a..807b6a27d8 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py @@ -126,7 +126,7 @@ class DsparkDraftSampler: # Gated/RNN subclasses return None (hidden-state-dependent bias); fall # through to the block sampler below. draft_tokens = None - fused_greedy = getattr(self.markov_head, "supports_sharded_greedy", False) or ( + fused_greedy = self.markov_head.supports_sharded_greedy or ( envs.SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV.get() and isinstance(self.markov_head, VanillaMarkov) )