[DSpark] Run every markov head through one step interface (#43620)

This commit is contained in:
Liangsheng Yin
2026-10-11 15:07:51 -07:00
committed by GitHub
parent d62b9c2480
commit 9943cf8eef
3 changed files with 97 additions and 174 deletions
+21 -51
View File
@@ -48,11 +48,10 @@ from sglang.srt.models.deepseek_v4 import (
make_hc_head_params,
)
from sglang.srt.models.dspark import (
BaseDSparkMarkovHead,
DSparkConfidenceHead,
StepSampler,
gather_and_crop_vocab,
project_through_lm_head,
run_markov_block,
)
from sglang.srt.runtime_context import (
get_parallel,
@@ -398,9 +397,7 @@ class MarkovW2ShardGeometry(msgspec.Struct, frozen=True):
num_embeddings_padded: int
class DSparkV4MarkovHead(nn.Module):
markov_head_type = "vanilla"
class DSparkV4MarkovHead(BaseDSparkMarkovHead):
def __init__(
self, *, vocab_size: int, markov_rank: int, is_dsv41: bool = False
) -> None:
@@ -487,38 +484,32 @@ class DSparkV4MarkovHead(nn.Module):
return F.linear(latent_states.to(weight.dtype), weight).float()
return F.linear(latent_states.float(), weight)
def compute_step_bias(
self, token_ids: torch.Tensor, hidden_states: Optional[torch.Tensor]
) -> torch.Tensor:
del hidden_states
return self.project_bias(self.get_prev_embeddings(token_ids))
def apply_step_logits(
def step_bias(
self,
logits: torch.Tensor,
prev_tokens: torch.Tensor,
*,
token_ids: torch.Tensor,
hidden_states: Optional[torch.Tensor],
) -> torch.Tensor:
if self._tp_shard is not None:
return self._apply_step_logits_sharded(
base_local=logits, token_ids=token_ids
)
return logits + self.compute_step_bias(token_ids, hidden_states)
def _apply_step_logits_sharded(
self, *, base_local: torch.Tensor, token_ids: torch.Tensor
) -> torch.Tensor:
state: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
del hidden_states
latent = self.get_prev_embeddings(prev_tokens)
shard = self._tp_shard
latent = self.get_prev_embeddings(token_ids)
if shard is None:
return self.project_bias(latent), state
weight_local = self.markov_w2.weight[
shard.org_vocab_start : shard.org_vocab_end
]
if self._opt_markov_w2_bf16:
bias = F.linear(latent.to(weight_local.dtype), weight_local)
else:
bias = F.linear(latent.float(), weight_local)
step_local = BuildStepLocal.execute(bias=bias, base_local=base_local)
# Keep the bias in BF16; the step add upcasts it to FP32.
return F.linear(latent.to(weight_local.dtype), weight_local), state
return F.linear(latent.float(), weight_local), state
def step_logits(
self, base_logits: torch.Tensor, bias: torch.Tensor
) -> torch.Tensor:
if self._tp_shard is None:
return base_logits + bias
step_local = BuildStepLocal.execute(bias=bias, base_local=base_logits)
full = self._vocab_gather(step_local)
return full[..., : self.vocab_size]
@@ -536,13 +527,10 @@ class DSparkV4MarkovHead(nn.Module):
)
shard = self._tp_shard
weight = self.markov_w2.weight[shard.org_vocab_start : shard.org_vocab_end]
prev = first_prev_tokens.long()
tokens = []
for step in range(base_logits.shape[1]):
latent = self.get_prev_embeddings(prev)
# Preserve the same BF16 GEMM rounding before the FP32 logits add.
bias = F.linear(latent.to(weight.dtype), weight)
bias, _ = self.step_bias(prev, hidden_states=None, state=None)
prev = sharded_greedy_step(
bias,
base_logits[:, step],
@@ -558,24 +546,6 @@ class DSparkV4MarkovHead(nn.Module):
logits = self.project_bias(embed)
return logits, embed
def sample_block(
self,
base_logits: torch.Tensor,
*,
first_prev_tokens: torch.Tensor,
hidden_states: Optional[torch.Tensor],
sampler: StepSampler,
collect_corrected: bool = True,
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
return run_markov_block(
self,
base_logits,
first_prev_tokens=first_prev_tokens,
hidden_states=hidden_states,
sampler=sampler,
collect_corrected=collect_corrected,
)
def build_dspark_v4_confidence_head(
*, config: DeepSeekV4Config, markov_rank: int
+75 -122
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import logging
from abc import ABC, abstractmethod
from typing import Callable, Iterable, Optional, Tuple
import torch
@@ -50,7 +51,7 @@ def project_through_lm_head(hidden: torch.Tensor, lm_head: nn.Module) -> torch.T
def run_markov_block(
head: nn.Module,
head: BaseDSparkMarkovHead,
base_logits: torch.Tensor,
*,
first_prev_tokens: torch.Tensor,
@@ -58,6 +59,7 @@ def run_markov_block(
sampler: StepSampler,
collect_corrected: bool = True,
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
state = head.init_state(hidden_states)
batch_size, proposal_len = base_logits.shape[:2]
if proposal_len == 0:
empty = torch.empty(batch_size, 0, dtype=torch.long, device=base_logits.device)
@@ -68,11 +70,10 @@ def run_markov_block(
prev_tokens = first_prev_tokens.long()
for step_idx in range(proposal_len):
step_hidden = None if hidden_states is None else hidden_states[:, step_idx, ...]
step_logits = head.apply_step_logits(
base_logits[:, step_idx, :],
token_ids=prev_tokens,
hidden_states=step_hidden,
bias, state = head.step_bias(
prev_tokens, hidden_states=step_hidden, state=state
)
step_logits = head.step_logits(base_logits[:, step_idx, :], bias)
next_tokens = sampler(step_logits, step_idx)
sampled_tokens.append(next_tokens)
if collect_corrected:
@@ -84,53 +85,34 @@ def run_markov_block(
)
class VanillaMarkov(nn.Module):
markov_head_type = "vanilla"
class BaseDSparkMarkovHead(nn.Module, ABC):
"""Corrects a draft block step by step: each step's base logits get a bias
computed from the token drafted at the step before."""
def __init__(self, *, vocab_size: int, markov_rank: int) -> None:
super().__init__()
self.vocab_size = int(vocab_size)
self.markov_rank = int(markov_rank)
if self.markov_rank <= 0:
raise ValueError(
f"VanillaMarkov requires markov_rank > 0, got {self.markov_rank}."
)
self.markov_w1 = nn.Embedding(self.vocab_size, self.markov_rank)
self.markov_w2 = nn.Linear(self.markov_rank, self.vocab_size, bias=False)
def get_prev_embeddings(self, token_ids: torch.Tensor) -> torch.Tensor:
return self.markov_w1(token_ids.long())
def project_bias(self, latent_states: torch.Tensor) -> torch.Tensor:
return self.markov_w2(latent_states)
def compute_step_bias(
self,
token_ids: torch.Tensor,
hidden_states: Optional[torch.Tensor],
) -> torch.Tensor:
def init_state(
self, hidden_states: Optional[torch.Tensor]
) -> Optional[torch.Tensor]:
del hidden_states
return self.project_bias(self.get_prev_embeddings(token_ids))
return None
def apply_step_logits(
@abstractmethod
def step_bias(
self,
logits: torch.Tensor,
prev_tokens: torch.Tensor,
*,
token_ids: torch.Tensor,
hidden_states: Optional[torch.Tensor],
) -> torch.Tensor:
return logits + self.compute_step_bias(token_ids, hidden_states)
state: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
"""Return this step's bias and the state the next step reads."""
def apply_block_logits(
self,
base_logits: torch.Tensor,
*,
token_ids: torch.Tensor,
hidden_states: Optional[torch.Tensor],
def step_logits(
self, base_logits: torch.Tensor, bias: torch.Tensor
) -> torch.Tensor:
if base_logits.size(-2) == 0:
return base_logits
return base_logits + self.compute_step_bias(token_ids, hidden_states)
return base_logits + bias
@property
def supports_sharded_greedy(self) -> bool:
return False
def sample_block(
self,
@@ -150,6 +132,35 @@ class VanillaMarkov(nn.Module):
collect_corrected=collect_corrected,
)
class VanillaMarkov(BaseDSparkMarkovHead):
def __init__(self, *, vocab_size: int, markov_rank: int) -> None:
super().__init__()
self.vocab_size = int(vocab_size)
self.markov_rank = int(markov_rank)
if self.markov_rank <= 0:
raise ValueError(
f"VanillaMarkov requires markov_rank > 0, got {self.markov_rank}."
)
self.markov_w1 = nn.Embedding(self.vocab_size, self.markov_rank)
self.markov_w2 = nn.Linear(self.markov_rank, self.vocab_size, bias=False)
def get_prev_embeddings(self, token_ids: torch.Tensor) -> torch.Tensor:
return self.markov_w1(token_ids.long())
def project_bias(self, latent_states: torch.Tensor) -> torch.Tensor:
return self.markov_w2(latent_states)
def step_bias(
self,
prev_tokens: torch.Tensor,
*,
hidden_states: Optional[torch.Tensor],
state: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
del hidden_states
return self.project_bias(self.get_prev_embeddings(prev_tokens)), state
def sample_block_greedy_fused(
self,
base_logits: torch.Tensor,
@@ -219,8 +230,6 @@ class Nemotron35VanillaMarkov(VanillaMarkov):
class GatedMarkovHead(VanillaMarkov):
markov_head_type = "gated"
def __init__(self, *, vocab_size: int, markov_rank: int, hidden_size: int) -> None:
super().__init__(vocab_size=vocab_size, markov_rank=markov_rank)
self.gate_proj = nn.Linear(int(hidden_size) + markov_rank, markov_rank)
@@ -236,16 +245,18 @@ class GatedMarkovHead(VanillaMarkov):
gate_inputs = torch.cat([hidden_states, prev_embeddings], dim=-1)
return torch.sigmoid(self.gate_proj(gate_inputs))
def compute_step_bias(
def step_bias(
self,
token_ids: torch.Tensor,
prev_tokens: torch.Tensor,
*,
hidden_states: Optional[torch.Tensor],
) -> torch.Tensor:
prev_embeddings = self.get_prev_embeddings(token_ids)
gate = self.compute_gate(token_ids, hidden_states).to(
state: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
prev_embeddings = self.get_prev_embeddings(prev_tokens)
gate = self.compute_gate(prev_tokens, hidden_states).to(
dtype=prev_embeddings.dtype
)
return self.project_bias(gate * prev_embeddings)
return self.project_bias(gate * prev_embeddings), state
def sample_block_greedy_fused(
self,
@@ -259,8 +270,6 @@ class GatedMarkovHead(VanillaMarkov):
class RNNHead(VanillaMarkov):
markov_head_type = "rnn"
def __init__(self, *, vocab_size: int, markov_rank: int, hidden_size: int) -> None:
super().__init__(vocab_size=vocab_size, markov_rank=markov_rank)
self.hidden_size = int(hidden_size)
@@ -281,84 +290,28 @@ class RNNHead(VanillaMarkov):
bias = self.project_bias(torch.tanh(output_raw))
return new_state, bias
def compute_step_bias(
self,
token_ids: torch.Tensor,
hidden_states: Optional[torch.Tensor],
) -> torch.Tensor:
def init_state(
self, hidden_states: Optional[torch.Tensor]
) -> Optional[torch.Tensor]:
if hidden_states is None:
raise ValueError("RNNHead requires hidden_states.")
prev_embeddings = self.get_prev_embeddings(token_ids)
state = torch.zeros_like(prev_embeddings)
_, bias = self._rnn_step(state, prev_embeddings, hidden_states)
return bias
def apply_block_logits(
self,
base_logits: torch.Tensor,
*,
token_ids: torch.Tensor,
hidden_states: Optional[torch.Tensor],
) -> torch.Tensor:
if hidden_states is None:
raise ValueError("RNNHead requires hidden_states.")
block_size = base_logits.size(-2)
if block_size == 0:
return base_logits
leading_shape = base_logits.shape[:-2]
state = torch.zeros(
*leading_shape,
return torch.zeros(
hidden_states.shape[0],
self.markov_rank,
device=base_logits.device,
device=hidden_states.device,
dtype=hidden_states.dtype,
)
output_logits = []
for k in range(block_size):
prev_emb = self.get_prev_embeddings(token_ids[..., k])
state, bias = self._rnn_step(state, prev_emb, hidden_states[..., k, :])
output_logits.append(base_logits[..., k, :] + bias)
return torch.stack(output_logits, dim=-2)
def sample_block(
def step_bias(
self,
base_logits: torch.Tensor,
prev_tokens: torch.Tensor,
*,
first_prev_tokens: torch.Tensor,
hidden_states: Optional[torch.Tensor],
sampler: StepSampler,
collect_corrected: bool = True,
state: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
if hidden_states is None:
raise ValueError("RNNHead requires hidden_states.")
batch_size, proposal_len = base_logits.shape[:2]
if proposal_len == 0:
empty = torch.empty(
batch_size, 0, dtype=torch.long, device=base_logits.device
)
return empty, base_logits
state = torch.zeros(
batch_size,
self.markov_rank,
device=base_logits.device,
dtype=hidden_states.dtype,
)
sampled_tokens = []
corrected_logits = []
prev_tokens = first_prev_tokens.long()
for step_idx in range(proposal_len):
prev_emb = self.get_prev_embeddings(prev_tokens)
state, bias = self._rnn_step(state, prev_emb, hidden_states[:, step_idx, :])
step_logits = base_logits[:, step_idx, :] + bias
next_tokens = sampler(step_logits, step_idx)
sampled_tokens.append(next_tokens)
if collect_corrected:
corrected_logits.append(step_logits.unsqueeze(1))
prev_tokens = next_tokens
return (
torch.stack(sampled_tokens, dim=1),
torch.cat(corrected_logits, dim=1) if collect_corrected else None,
)
prev_emb = self.get_prev_embeddings(prev_tokens)
state, bias = self._rnn_step(state, prev_emb, hidden_states)
return bias, state
def sample_block_greedy_fused(
self,
@@ -126,7 +126,7 @@ class DsparkDraftSampler:
# Gated/RNN subclasses return None (hidden-state-dependent bias); fall
# through to the block sampler below.
draft_tokens = None
fused_greedy = getattr(self.markov_head, "supports_sharded_greedy", False) or (
fused_greedy = self.markov_head.supports_sharded_greedy or (
envs.SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV.get()
and isinstance(self.markov_head, VanillaMarkov)
)