[Spec] Add LiLiCorr: a candidate-lattice reranker for DFlash drafts (#37462)

Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: kpham-sgl <khoa.pham@radixark.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
mrusanovsky
2026-09-27 15:36:48 -07:00
committed by GitHub
co-authored by Po-Han Huang kpham-sgl Claude Opus 5.5
parent 14ab74ef95
commit 78eee88113
11 changed files with 1551 additions and 47 deletions
@@ -21,6 +21,8 @@ _TRITON_KERNELS = [
("topk1", "draft_topk1_postprocess"),
("ragged_verify_kernels", "pad_verify_lens_to_bucket"),
("ragged_verify_kernels", "build_qo_indptr"),
("lilicorr", "lilicorr_topk_lse"),
("lilicorr", "lilicorr_sample_path"),
("reject_sampling", "chain_speculative_sampling_triton"),
("row_argmax", "row_argmax"),
("reconstruct_tree", "reconstruct_indices_from_tree_mask_triton"),
@@ -0,0 +1,235 @@
from __future__ import annotations
from typing import Tuple
import torch
import torch.nn.functional as F
import triton
import triton.language as tl
from sglang.kernels.ops.speculative.dflash import selector_walk_triton
# Launch geometry tuned on H100 for a ~150k vocabulary; TILE * TPP is the load block.
_TILE = 1024
_TILES_PER_PROGRAM = 8
_NUM_WARPS = 4
# One Triton lane group in the selector walk; the config refuses a wider head.
MAX_FUSED_CANDIDATE_TOPK = 16
_NEG = -3.0e38
@triton.jit
def _tiled_scan(
lp, # [N, V] logits (any float dtype)
tmax, # [N, T] fp32 per-tile max
pm, # [N, P] fp32 online-softmax running max
ps, # [N, P] fp32 online-softmax running sum
V,
T,
stride_lp_n,
stride_tmax_n,
stride_p_n,
TILE: tl.constexpr,
TPP: tl.constexpr,
):
neg = -3.0e38
row = tl.program_id(0)
grp = tl.program_id(1)
t0 = grp * TPP
tile_off = t0 + tl.arange(0, TPP)
offs = tile_off[:, None] * TILE + tl.arange(0, TILE)[None, :]
mask = (tile_off[:, None] < T) & (offs < V)
x = tl.load(lp + row * stride_lp_n + offs, mask=mask, other=neg)
x = x.to(tl.float32)
tm = tl.max(x, axis=1)
tl.store(tmax + row * stride_tmax_n + tile_off, tm, mask=tile_off < T)
m = tl.max(tm, axis=0)
s = tl.sum(tl.exp(x - m), axis=0)
s = tl.sum(s, axis=0)
tl.store(pm + row * stride_p_n + grp, m)
tl.store(ps + row * stride_p_n + grp, s)
@triton.jit
def _tiled_select(
lp, # [N, V] logits
tids, # [N, KT] int64 candidate tile ids (ascending)
ov, # [N, K] fp32 out vals (descending)
oi, # [N, K] int64 out ids
V,
stride_lp_n,
stride_tid_n,
stride_ov_n,
stride_oi_n,
TILE: tl.constexpr,
KT: tl.constexpr,
K: tl.constexpr,
NSEL: tl.constexpr,
):
neg = -3.0e38
row = tl.program_id(0)
tid = tl.load(tids + row * stride_tid_n + tl.arange(0, KT))
offs2 = tid[:, None] * TILE + tl.arange(0, TILE)[None, :]
mask2 = offs2 < V
x2 = tl.load(lp + row * stride_lp_n + offs2, mask=mask2, other=neg)
x = tl.reshape(x2.to(tl.float32), (NSEL,))
ids = tl.reshape(offs2, (NSEL,))
lane = tl.arange(0, NSEL)
for j in range(K):
mv = tl.max(x, axis=0)
mp = tl.argmax(x, axis=0)
mid = tl.sum(tl.where(lane == mp, ids, 0), axis=0)
tl.store(ov + row * stride_ov_n + j, mv)
tl.store(oi + row * stride_oi_n + j, mid)
x = tl.where(lane == mp, neg, x)
def _combine_lse(pm: torch.Tensor, ps: torch.Tensor) -> torch.Tensor:
gm = pm.max(dim=-1, keepdim=True).values
s = (ps * (pm - gm).exp()).sum(dim=-1)
return gm.squeeze(-1) + s.log()
def _topk_lse_torch(
logits: torch.Tensor, k: int
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
vals, ids = torch.topk(logits, k, dim=-1)
rowmax = vals[:, 0:1] # topk is sorted descending
sumexp = (logits - rowmax).exp().sum(dim=-1, dtype=torch.float32)
lse = rowmax.squeeze(-1).to(torch.float32) + sumexp.log()
return vals.float(), ids.to(torch.int64), lse
def lilicorr_topk_lse(
logits: torch.Tensor, k: int
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
# Tile pre-selection is exact: a top-k element's tile has a max at least as large,
# so it is among the k largest-max tiles.
if not logits.is_cuda:
return _topk_lse_torch(logits, k)
logits = logits.contiguous()
n, V = int(logits.shape[0]), int(logits.shape[1])
K = int(k)
T = (V + _TILE - 1) // _TILE
if min(K, T) & (min(K, T) - 1):
return _topk_lse_torch(logits, k)
P = (T + _TILES_PER_PROGRAM - 1) // _TILES_PER_PROGRAM
device = logits.device
tmax = torch.full((n, T), _NEG, dtype=torch.float32, device=device)
pm = torch.full((n, P), _NEG, dtype=torch.float32, device=device)
ps = torch.zeros((n, P), dtype=torch.float32, device=device)
_tiled_scan[(n, P)](
logits,
tmax,
pm,
ps,
V,
T,
logits.stride(0),
tmax.stride(0),
pm.stride(0),
TILE=_TILE,
TPP=_TILES_PER_PROGRAM,
num_warps=_NUM_WARPS,
)
kt = min(K, T)
tids = torch.topk(tmax, kt, dim=-1).indices
tids = torch.sort(tids, dim=-1).values.to(torch.int64)
ov = torch.empty((n, K), dtype=torch.float32, device=device)
oi = torch.empty((n, K), dtype=torch.int64, device=device)
_tiled_select[(n,)](
logits,
tids,
ov,
oi,
V,
logits.stride(0),
tids.stride(0),
ov.stride(0),
oi.stride(0),
TILE=_TILE,
KT=kt,
K=K,
NSEL=kt * _TILE,
num_warps=_NUM_WARPS,
)
return ov, oi, _combine_lse(pm, ps)
def _lattice_scores(log_start: torch.Tensor, log_pair: torch.Tensor) -> torch.Tensor:
# The walk reads only scores[:, 0, 0, :] at slot 0, so broadcasting is sound.
topk = int(log_start.shape[-1])
start = log_start.float()[:, None, None, :].expand(-1, 1, topk, topk)
return torch.cat([start, log_pair.float()], dim=1)
def _selector_walk_torch(
*,
candidate_ids: torch.Tensor,
scores: torch.Tensor,
uniforms: torch.Tensor,
temperatures: torch.Tensor,
greedy_mask: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
_, num_slots, topk = candidate_ids.shape
temps = temperatures.view(-1, 1).to(torch.float32)
greedy = greedy_mask.view(-1)
previous = torch.zeros(greedy.shape[0], dtype=torch.int64, device=scores.device)
tokens, q_rows = [], []
for slot in range(num_slots):
node = torch.gather(
scores[:, slot].float(), 1, previous.view(-1, 1, 1).expand(-1, 1, topk)
).squeeze(1)
probs = torch.softmax(node / temps, dim=-1)
sampled = (
uniforms[:, slot : slot + 1]
.ge(probs.cumsum(dim=-1))
.sum(dim=-1)
.clamp_max(topk - 1)
)
previous = torch.where(greedy, node.argmax(dim=-1), sampled)
q_rows.append(
torch.where(
greedy.unsqueeze(-1),
F.one_hot(previous, topk).to(torch.float32),
probs,
)
)
tokens.append(
torch.gather(candidate_ids[:, slot], 1, previous.view(-1, 1)).squeeze(1)
)
return torch.stack(tokens, dim=-1).to(torch.int64), torch.stack(q_rows, dim=1)
def lilicorr_sample_path(
log_start: torch.Tensor,
log_pair: torch.Tensor,
candidate_tokens: torch.Tensor,
uniforms: torch.Tensor,
temperatures: torch.Tensor,
greedy_mask: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
# No log-prob prior in the proposal: the head was trained without one. Greedy rows
# report a point mass so min(1, p/q) stays the right acceptance test.
scores = _lattice_scores(log_start, log_pair)
walk = selector_walk_triton if scores.is_cuda else _selector_walk_torch
return walk(
candidate_ids=candidate_tokens,
scores=scores,
uniforms=uniforms,
temperatures=temperatures,
greedy_mask=greedy_mask,
)
+2
View File
@@ -1366,6 +1366,8 @@ class Envs:
SGLANG_OPT_FUSED_KDA_VERIFY = EnvBool(False)
# A/B: keep the DFLASH draft greedy head eager (not folded in-graph).
SGLANG_DFLASH_EAGER_DRAFT_SAMPLER = EnvBool(False)
SGLANG_ENABLE_LILICORR_SAMPLING = EnvBool(False)
SGLANG_LILICORR_REQUIRE_SAMPLING = EnvBool(False)
SGLANG_RAGGED_VERIFY_MODE = EnvStr("static")
SGLANG_TEST_RAGGED_VERIFY_FORCE_UNIFORM_CAPTURE = EnvBool(False)
# Skip draft_extend while adaptive spec is at steps=0 (drafting disabled).
+57 -39
View File
@@ -12,6 +12,7 @@ import torch.nn.functional as F
from torch import nn
from sglang.kernels.ops.speculative.dflash import selector_walk_triton
from sglang.kernels.ops.speculative.lilicorr import lilicorr_topk_lse
from sglang.srt.configs.laguna import normalize_gating
from sglang.srt.distributed.communication_op import tensor_model_parallel_all_gather
from sglang.srt.layers.activation import SiluAndMul
@@ -101,6 +102,56 @@ def _project_candidate_logits(
return logits
def candidate_topk(
hidden: torch.Tensor,
lm_head: nn.Module,
k: int,
*,
with_partition: bool = False,
) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
"""Global top-k candidates from the target head: hidden [N, H] -> ids / vals [N, K].
Under TP (vocab-sharded lm_head): local top-k per shard, all-gather K logits/ids (not
the full vocab), then a global top-k -- identical candidates at O(tp*K) instead of
O(vocab) gather bandwidth. with_partition also returns the full-vocabulary lse [N] fp32.
"""
# The worker screens the head before capture, but the eager fallbacks attach
# whatever the target has.
weight = getattr(lm_head, "weight", None)
quant_method = getattr(lm_head, "quant_method", None)
use_quant_head = should_apply_lm_head_quant_method(lm_head, quant_method)
if not use_quant_head and not is_dense_head_weight(weight):
raise RuntimeError(
"DFlash candidate top-k requires a dense FP16/BF16/FP32 target lm_head "
"or a supported lm_head.quant_method."
)
tp = get_parallel().tp_size == 1
num_org = (
int(lm_head.org_vocab_size)
if tp
else int(lm_head.shard_indices.num_org_elements)
)
logits = _project_candidate_logits(
hidden, lm_head, num_org=num_org, use_quant_head=use_quant_head
)
lse = None
if with_partition:
vals, ids, lse = lilicorr_topk_lse(logits, k)
else:
vals, ids = _radix_topk(logits, k)
if tp:
return ids.long(), vals, lse
global_ids = ids.long() + int(lm_head.shard_indices.org_vocab_start_index)
gathered_vals = tensor_model_parallel_all_gather(vals.float(), dim=-1)
gathered_ids = tensor_model_parallel_all_gather(global_ids, dim=-1)
top_vals, sel = torch.topk(gathered_vals, k, dim=-1)
if lse is not None:
lse = torch.logsumexp(
tensor_model_parallel_all_gather(lse.unsqueeze(-1), dim=-1), dim=-1
)
return torch.gather(gathered_ids, -1, sel).long(), top_vals, lse
def _get_dflash_attention_type(config, *, default: AttentionType) -> AttentionType:
"""Honor explicit causality while preserving legacy layer defaults."""
text_config = config.get_text_config()
@@ -579,11 +630,13 @@ class DFlashDraftModel(nn.Module):
hidden_size = int(config.hidden_size)
num_layers = int(config.num_hidden_layers)
rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-6))
self.rms_norm_eps = rms_norm_eps
draft_config = self.draft_config = parse_dflash_draft_config(
draft_hf_config=config
)
self.block_size = draft_config.resolve_block_size(default=16)
self.candidate_selector: Optional[nn.Module] = None
self.lilicorr: Optional[nn.Module] = None
self.is_nemotron_35_draft = is_nemotron_35_draft_config(config)
self.embed_tokens: Optional[VocabParallelEmbedding] = None
if self.is_nemotron_35_draft:
@@ -1176,47 +1229,12 @@ class DFlash2DraftModel(DFlashDraftModel):
self, hidden: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Top-k base candidates via the target lm_head: hidden [N, H] -> global
candidate_ids / unary_logits [N, K]. Under TP (vocab-sharded lm_head): local top-k
per shard, all-gather K logits/ids (not the full vocab), then a global top-k --
identical candidates at O(tp*K) instead of O(vocab) gather bandwidth."""
candidate_ids / unary_logits [N, K]."""
assert self.lm_head is not None, "draft_model.lm_head unset before capture"
k = self.candidate_selector.top_k
# The worker screens the head before capture, but its eager fallback
# (_propose_selector_block) attaches whatever the target has.
weight = getattr(self.lm_head, "weight", None)
quant_method = getattr(self.lm_head, "quant_method", None)
use_quant_head = should_apply_lm_head_quant_method(self.lm_head, quant_method)
if not use_quant_head and not is_dense_head_weight(weight):
raise RuntimeError(
"DFlash2 selector requires a dense FP16/BF16/FP32 target lm_head "
"or a supported lm_head.quant_method."
)
if get_parallel().tp_size == 1:
org = int(self.lm_head.org_vocab_size)
vals, ids = _radix_topk(
_project_candidate_logits(
hidden, self.lm_head, num_org=org, use_quant_head=use_quant_head
),
k,
)
return ids.long(), self._transform_unary_logits(vals)
shard = self.lm_head.shard_indices
vals, ids = _radix_topk(
_project_candidate_logits(
hidden,
self.lm_head,
num_org=int(shard.num_org_elements),
use_quant_head=use_quant_head,
),
k,
)
global_ids = ids.long() + int(shard.org_vocab_start_index)
gathered_vals = tensor_model_parallel_all_gather(vals.float(), dim=-1)
gathered_ids = tensor_model_parallel_all_gather(global_ids, dim=-1)
top_vals, sel = torch.topk(gathered_vals, k, dim=-1)
return torch.gather(gathered_ids, -1, sel).long(), self._transform_unary_logits(
top_vals
ids, vals, _ = candidate_topk(
hidden, self.lm_head, self.candidate_selector.top_k
)
return ids, self._transform_unary_logits(vals)
class MuseGlimmerAssistantModel(DFlashDraftModel):
+473
View File
@@ -0,0 +1,473 @@
from __future__ import annotations
import logging
from typing import Iterable, Optional, Tuple
import torch
import torch.nn.functional as F
from torch import nn
from sglang.kernels.ops.speculative.lilicorr import lilicorr_sample_path
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.models.dflash import DFlashDraftModel
from sglang.srt.speculative.lilicorr_utils import (
LiLiCorrConfig,
parse_lilicorr_draft_config,
)
logger = logging.getLogger(__name__)
class LiLiCorrLatticeAttention(nn.Module):
# nn.MultiheadAttention's parameter layout, run via SDPA since that module is not capturable.
def __init__(self, hidden_size: int, num_heads: int) -> None:
super().__init__()
if hidden_size % num_heads != 0:
raise ValueError(
f"LiLiCorr hidden_size={hidden_size} must be divisible by "
f"num_heads={num_heads}."
)
self.hidden_size = int(hidden_size)
self.num_heads = int(num_heads)
self.head_dim = self.hidden_size // self.num_heads
self.in_proj_weight = nn.Parameter(torch.zeros(3 * hidden_size, hidden_size))
self.in_proj_bias = nn.Parameter(torch.zeros(3 * hidden_size))
self.out_proj = nn.Linear(hidden_size, hidden_size)
def forward(
self, hidden_states: torch.Tensor, attention_bias: torch.Tensor
) -> torch.Tensor:
bsz, seq_len, _ = hidden_states.shape
qkv = F.linear(hidden_states, self.in_proj_weight, self.in_proj_bias)
q, k, v = qkv.chunk(3, dim=-1)
shape = (bsz, seq_len, self.num_heads, self.head_dim)
q = q.view(shape).transpose(1, 2)
k = k.view(shape).transpose(1, 2)
v = v.view(shape).transpose(1, 2)
# attention_bias is [1, heads, L, L]; SDPA broadcasts it over the batch.
out = F.scaled_dot_product_attention(q, k, v, attn_mask=attention_bias)
return self.out_proj(
out.transpose(1, 2).reshape(bsz, seq_len, self.hidden_size)
)
class LiLiCorrLayer(nn.Module):
def __init__(
self,
hidden_size: int,
num_heads: int,
mlp_ratio: float,
rms_norm_eps: float,
) -> None:
super().__init__()
self.attn_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
self.attn = LiLiCorrLatticeAttention(hidden_size, num_heads)
self.mlp_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
mlp_hidden_size = int(hidden_size * mlp_ratio)
self.mlp = nn.Sequential(
nn.Linear(hidden_size, mlp_hidden_size),
nn.SiLU(),
nn.Linear(mlp_hidden_size, hidden_size),
)
def forward(
self, hidden_states: torch.Tensor, attention_bias: torch.Tensor
) -> torch.Tensor:
hidden_states = hidden_states + self.attn(
self.attn_norm(hidden_states), attention_bias
)
return hidden_states + self.mlp(self.mlp_norm(hidden_states))
class LiLiCorrHead(nn.Module):
# [log_probs, probs, logprob_gap, rank_frac, is_top1], in the trained Linear's order.
num_candidate_features = 5
def __init__(
self,
*,
model_hidden_size: int,
block_size: int,
rms_norm_eps: float,
config: LiLiCorrConfig,
) -> None:
super().__init__()
hidden_size = config.resolve_hidden_size(model_hidden_size=model_hidden_size)
self.block_size = int(block_size)
self.num_candidate_slots = self.block_size - 1
self.num_active_slots = self.num_candidate_slots
self.candidate_topk = int(config.candidate_topk)
self.hidden_size = hidden_size
self.num_heads = int(config.num_heads)
self.mlp_ratio = float(config.mlp_ratio)
self.factor_dim = int(config.factor_dim)
self.vector_eps = float(config.vector_eps)
self.logit_scale = float(config.logit_scale)
self.token_proj = (
nn.Identity()
if model_hidden_size == hidden_size
else nn.Linear(model_hidden_size, hidden_size)
)
self.pass_hidden_proj = nn.Linear(model_hidden_size, hidden_size)
self.feature_mlp = nn.Sequential(
nn.LayerNorm(self.num_candidate_features),
nn.Linear(self.num_candidate_features, hidden_size),
nn.SiLU(),
nn.Linear(hidden_size, hidden_size),
)
self.slot_embedding = nn.Parameter(
torch.zeros(1, 1, self.num_candidate_slots, 1, hidden_size)
)
self.rank_embedding = nn.Parameter(
torch.zeros(1, 1, 1, self.candidate_topk, hidden_size)
)
self.relative_slot_bias = nn.Parameter(
torch.zeros(self.num_heads, 2 * self.block_size - 1)
)
self.same_slot_bias = nn.Parameter(torch.zeros(self.num_heads))
self.context_proj = nn.Linear(model_hidden_size, hidden_size)
self.layers = nn.ModuleList(
[
LiLiCorrLayer(
hidden_size=hidden_size,
num_heads=self.num_heads,
mlp_ratio=self.mlp_ratio,
rms_norm_eps=rms_norm_eps,
)
for _ in range(int(config.num_layers))
]
)
self.output_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
self.anchor_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
self.factor_input_proj = nn.Linear(hidden_size * 3, hidden_size)
self.out_head = nn.Linear(hidden_size, self.factor_dim)
self.in_head = nn.Linear(hidden_size, self.factor_dim)
self.anchor_out_head = nn.Linear(hidden_size, self.factor_dim)
self._attn_bias: Optional[torch.Tensor] = None
self._fused_edge_weight: Optional[torch.Tensor] = None
self._fused_edge_bias: Optional[torch.Tensor] = None
self._factor_input_splits: Optional[Tuple[torch.Tensor, ...]] = None
self._rank_frac_col: Optional[torch.Tensor] = None
self._is_top1_col: Optional[torch.Tensor] = None
@torch.no_grad()
def materialize_inference_buffers(
self, device: torch.device, dtype: torch.dtype
) -> None:
# After weight load and before graph capture: host work, not capturable.
topk = self.candidate_topk
self._attn_bias = self._build_attention_bias(device=device, dtype=dtype)
self._fused_edge_weight = (
torch.cat([self.out_head.weight, self.in_head.weight], dim=0)
.to(device=device, dtype=dtype)
.contiguous()
)
self._fused_edge_bias = (
torch.cat([self.out_head.bias, self.in_head.bias], dim=0)
.to(device=device, dtype=dtype)
.contiguous()
)
weight = self.factor_input_proj.weight
hdim = self.hidden_size
self._factor_input_splits = (
weight[:, :hdim].contiguous(),
weight[:, hdim : 2 * hdim].contiguous(),
weight[:, 2 * hdim :].contiguous(),
)
if topk > 1:
rank_frac = torch.arange(topk, device=device, dtype=torch.float32).view(
1, 1, 1, topk
) / float(topk - 1)
else:
rank_frac = torch.zeros(1, 1, 1, topk, device=device, dtype=torch.float32)
is_top1 = torch.zeros(1, 1, 1, topk, device=device, dtype=torch.float32)
is_top1[..., 0] = 1.0
self._rank_frac_col = rank_frac.contiguous()
self._is_top1_col = is_top1.contiguous()
def _build_attention_bias(
self, *, device: torch.device, dtype: torch.dtype
) -> torch.Tensor:
topk = self.candidate_topk
slot_ids = torch.arange(
self.num_active_slots, device=device, dtype=torch.long
).repeat_interleave(topk)
rel = slot_ids.view(-1, 1) - slot_ids.view(1, -1)
rel = rel.clamp(min=-(self.block_size - 1), max=self.block_size - 1)
bias = self.relative_slot_bias[:, rel + self.block_size - 1]
same_slot = slot_ids.view(-1, 1) == slot_ids.view(1, -1)
bias = bias + same_slot.unsqueeze(0).to(dtype=bias.dtype) * (
self.same_slot_bias.view(-1, 1, 1)
)
return bias.to(device=device, dtype=dtype).contiguous()
def set_active_block_size(self, block_size: int) -> None:
# A prefix of the trained slots; every relative offset it needs was trained.
if not 2 <= int(block_size) <= self.block_size:
raise ValueError(
f"block_size={int(block_size)} is outside [2, {self.block_size}], "
"the trained LiLiCorr block."
)
self.num_active_slots = int(block_size) - 1
if self._attn_bias is not None:
self._attn_bias = self._build_attention_bias(
device=self._attn_bias.device, dtype=self._attn_bias.dtype
)
def _require_materialized(self) -> None:
if self._attn_bias is None:
raise RuntimeError(
"LiLiCorr head scored before materialize_inference_buffers(): its "
"attention bias and fused edge heads are unbuilt."
)
def _project_anchor(
self, anchor_hidden: torch.Tensor, anchor_valid: torch.Tensor
) -> torch.Tensor:
# Branch-free so there is no host sync inside the captured region.
anchor = self.context_proj(anchor_hidden)
return anchor * anchor_valid.unsqueeze(-1).to(anchor.dtype)
def score(
self,
*,
token_embeddings: torch.Tensor,
candidate_log_probs: torch.Tensor,
pass_hidden: torch.Tensor,
anchor_hidden: torch.Tensor,
anchor_valid: torch.Tensor,
already_projected: bool = False,
) -> Tuple[torch.Tensor, torch.Tensor]:
self._require_materialized()
bsz, n_blocks, n_slots, topk = candidate_log_probs.shape
if topk != self.candidate_topk:
raise ValueError(
f"LiLiCorr was built for candidate_topk={self.candidate_topk} but the "
f"lattice carries {topk}; both are sized for the trained width."
)
proj_dtype = self.pass_hidden_proj.weight.dtype
if token_embeddings.dtype != proj_dtype:
token_embeddings = token_embeddings.to(proj_dtype)
if pass_hidden.dtype != proj_dtype:
pass_hidden = pass_hidden.to(proj_dtype)
token_states = (
token_embeddings if already_projected else self.token_proj(token_embeddings)
)
pass_states = self.pass_hidden_proj(pass_hidden).unsqueeze(-2)
log_probs = candidate_log_probs.float()
features = torch.stack(
[
log_probs,
log_probs.exp(),
log_probs - log_probs.max(dim=-1, keepdim=True).values,
self._rank_frac_col.expand_as(log_probs),
self._is_top1_col.expand_as(log_probs),
],
dim=-1,
)
hidden_states = token_states + pass_states
hidden_states = hidden_states + self.feature_mlp(
features.to(dtype=token_states.dtype)
)
hidden_states = hidden_states + self.slot_embedding[:, :, :n_slots]
hidden_states = hidden_states + self.rank_embedding
hidden_states = hidden_states.reshape(
bsz * n_blocks, n_slots * topk, self.hidden_size
)
anchor_state = self._project_anchor(anchor_hidden, anchor_valid)
attention_bias = self._attn_bias.unsqueeze(0)
for layer in self.layers:
hidden_states = layer(hidden_states, attention_bias)
hidden_states = self.output_norm(hidden_states).reshape(
bsz, n_blocks, n_slots, topk, self.hidden_size
)
anchor_state = self.anchor_norm(anchor_state)
w_self, w_anchor, w_cross = self._factor_input_splits
anchor_row = anchor_state[:, :, None, None, :]
pre = F.linear(hidden_states, w_self, self.factor_input_proj.bias)
pre = pre + F.linear(anchor_row, w_anchor)
pre = pre + F.linear(hidden_states * anchor_row, w_cross)
factor_hidden = F.silu(pre)
edges = F.linear(factor_hidden, self._fused_edge_weight, self._fused_edge_bias)
out_vec, in_vec = F.normalize(
edges.unflatten(-1, (2, self.factor_dim)),
dim=-1,
eps=self.vector_eps,
).unbind(-2)
anchor_out = F.normalize(
self.anchor_out_head(anchor_state), dim=-1, eps=self.vector_eps
)
start_scores = (anchor_out[:, :, None, :] * in_vec[:, :, 0, :, :]).sum(dim=-1)
pair_scores = torch.matmul(
out_vec[:, :, :-1], in_vec[:, :, 1:].transpose(-1, -2)
)
return start_scores, pair_scores
def log_factors(
self, start_scores: torch.Tensor, pair_scores: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
return (
self.logit_scale * start_scores.float(),
self.logit_scale * pair_scores.float(),
)
def select_with_proposal(
self,
*,
token_embeddings: torch.Tensor,
candidate_tokens: torch.Tensor,
candidate_log_probs: torch.Tensor,
pass_hidden: torch.Tensor,
anchor_hidden: torch.Tensor,
anchor_valid: torch.Tensor,
uniforms: torch.Tensor,
temperatures: torch.Tensor,
greedy_mask: torch.Tensor,
already_projected: bool = False,
) -> Tuple[torch.Tensor, torch.Tensor]:
start_scores, pair_scores = self.score(
token_embeddings=token_embeddings.unsqueeze(1),
candidate_log_probs=candidate_log_probs.unsqueeze(1),
pass_hidden=pass_hidden.unsqueeze(1),
anchor_hidden=anchor_hidden.unsqueeze(1),
anchor_valid=anchor_valid.unsqueeze(1),
already_projected=already_projected,
)
log_start, log_pair = self.log_factors(start_scores, pair_scores)
return lilicorr_sample_path(
log_start[:, 0, :],
log_pair[:, 0],
candidate_tokens,
uniforms=uniforms,
temperatures=temperatures,
greedy_mask=greedy_mask,
)
@torch.no_grad()
def build_token_table(self, embed_tokens: nn.Module) -> Optional[torch.Tensor]:
if isinstance(self.token_proj, nn.Identity):
return None
weight = embed_tokens.weight
if int(self.token_proj.weight.shape[1]) != int(weight.shape[1]):
return None
return F.linear(
weight.to(self.token_proj.weight.dtype),
self.token_proj.weight,
self.token_proj.bias,
).contiguous()
def check_head_weight_coverage(head: LiLiCorrHead, seen: set) -> None:
# The base loader silently drops unresolved weights, which would only show
# up as a lower acceptance length.
expected = {f"lilicorr.{name}" for name, _ in head.named_parameters()}
missing = sorted(expected - seen)
if missing:
raise ValueError(
f"LiLiCorr checkpoint is missing {len(missing)} head parameters "
f"(e.g. {missing[:5]}); a draft with no LiLiCorr head should declare "
'architectures=["DFlashDraftModel"].'
)
unexpected = sorted(seen - expected)
if unexpected:
raise ValueError(
f"LiLiCorr checkpoint carries {len(unexpected)} head tensors this head "
f"has no parameter for (e.g. {unexpected[:5]}). Check "
"lilicorr_hidden_size, lilicorr_num_layers and lilicorr_factor_dim in "
"dflash_config against the checkpoint."
)
def check_conv_weight_coverage(model: DFlashDraftModel, seen: set) -> None:
# conv_kernel_size / conv_group_size default to 0, so a config that lost them
# builds no conv modules and the loader drops every conv tensor silently.
expected = {
name
for name, _ in model.named_parameters()
if ".attention_conv." in name or ".mlp_conv." in name
}
if seen and not expected:
raise ValueError(
f"Draft checkpoint carries {len(seen)} grouped-convolution tensors "
f"(e.g. {sorted(seen)[:3]}) but this draft built no convolution modules: "
"dflash_config is missing conv_kernel_size / conv_group_size, which both "
"default to 0 and cannot be inferred from the tensors."
)
if expected and not seen:
raise ValueError(
f"This draft built {len(expected)} grouped-convolution parameters from "
"dflash_config, but the checkpoint carries none, so kernel_projection "
"would serve at its random initialization."
)
missing = sorted(expected - seen)
unexpected = sorted(seen - expected)
if missing or unexpected:
raise ValueError(
"Draft checkpoint's grouped-convolution tensors do not correspond to the "
f"built ones: {len(missing)} missing (e.g. {missing[:3]}), "
f"{len(unexpected)} unexpected (e.g. {unexpected[:3]}). Check "
"conv_kernel_size, conv_group_size and num_hidden_layers in dflash_config."
)
if expected:
conv = model.layers[0].attention_conv
logger.info(
"DFLASH grouped convolution live: %d taps, group size %d, %d tensors.",
int(conv.taps),
int(conv.group_size),
len(expected),
)
class LiLiCorrDraftModel(DFlashDraftModel):
def __init__(self, config, quant_config=None, prefix: str = "") -> None:
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
self.lilicorr = LiLiCorrHead(
model_hidden_size=int(config.hidden_size),
block_size=int(self.block_size),
rms_norm_eps=self.rms_norm_eps,
config=parse_lilicorr_draft_config(draft_hf_config=config),
)
def set_block_size(self, block_size: int) -> None:
super().set_block_size(block_size)
self.lilicorr.set_active_block_size(block_size)
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
seen: set[str] = set()
seen_conv: set[str] = set()
def tracking():
for name, weight in weights:
stripped = name[len("model.") :] if name.startswith("model.") else name
if stripped.startswith("lilicorr."):
seen.add(stripped)
elif ".attention_conv." in stripped or ".mlp_conv." in stripped:
seen_conv.add(stripped)
yield name, weight
super().load_weights(tracking())
check_head_weight_coverage(self.lilicorr, seen)
check_conv_weight_coverage(self, seen_conv)
parameter = next(self.lilicorr.parameters())
self.lilicorr.materialize_inference_buffers(parameter.device, parameter.dtype)
EntryClass = [LiLiCorrDraftModel]
@@ -74,6 +74,13 @@ from sglang.srt.speculative.draft_worker_common import (
make_draft_sampler_capture_hook,
)
from sglang.srt.speculative.dspark_components.dspark_draft import resolve_greedy_mask
from sglang.srt.speculative.lilicorr_utils import (
build_lilicorr_draft_sampler,
propose_lilicorr_block,
publish_anchor,
resolve_sampling_enabled,
target_input_embeddings,
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite
from sglang.srt.speculative.spec_utils import (
@@ -386,6 +393,7 @@ class DFlashWorkerV2(BaseSpecWorker):
self.device = target_worker.device
self._warned_sampling_fallback = False
self._warned_lilicorr_eager = False
self._draft_probs_buf = None
self._logged_first_verify = False
self._full_embed_gpu: Optional[torch.Tensor] = None
@@ -425,6 +433,12 @@ class DFlashWorkerV2(BaseSpecWorker):
self.selector = self.draft_model.candidate_selector
# Ascend keeps selector proposal aligned with its greedy-only verify path.
self._selector_sampling_enabled = not _is_npu
self.lilicorr = self.draft_model.lilicorr
# The sampled commit rides the selector's accept path.
self._lilicorr_sampling_enabled = self.lilicorr is not None and (
resolve_sampling_enabled(device_supported=self._selector_sampling_enabled)
)
self._lilicorr_anchor: Optional[torch.Tensor] = None
draft_config = parse_dflash_draft_config(
draft_hf_config=self.draft_model_runner.model_config.hf_config
)
@@ -805,15 +819,17 @@ class DFlashWorkerV2(BaseSpecWorker):
if lm_head is None:
return _eager("no target lm_head")
# A gate-admitted quantized head is capture-safe: the target's own
# logits path already runs the same kernel under CUDA graphs.
head_supported = is_dense_head_weight(
getattr(lm_head, "weight", None)
) or should_apply_lm_head_quant_method(
lm_head, getattr(lm_head, "quant_method", None)
)
if self.selector is not None:
# compute_candidates needs the target lm_head attached before capture.
# A gate-admitted quantized head is capture-safe: the target's own
# logits path already runs the same kernel under CUDA graphs.
if not is_dense_head_weight(
getattr(lm_head, "weight", None)
) and not should_apply_lm_head_quant_method(
lm_head, getattr(lm_head, "quant_method", None)
):
if not head_supported:
return _eager("unsupported quantized lm_head")
self.draft_model.lm_head = lm_head
if self.model_runner.tp_rank == 0:
@@ -829,6 +845,17 @@ class DFlashWorkerV2(BaseSpecWorker):
device=self.device,
sampling_enabled=self._selector_sampling_enabled,
)
if self.lilicorr is not None:
if not head_supported:
return _eager("unsupported quantized lm_head")
return build_lilicorr_draft_sampler(
sampling_enabled=self._lilicorr_sampling_enabled,
head=self.lilicorr,
draft_model=self.draft_model,
embed_tokens=target_input_embeddings(target_model),
lm_head=lm_head,
block_size=self.block_size,
)
if not hasattr(lm_head, "weight"):
return _eager("quantized lm_head has no dense weight")
if not is_dense_head_weight(lm_head.weight):
@@ -1736,6 +1763,7 @@ class DFlashWorkerV2(BaseSpecWorker):
positions: torch.Tensor,
cache_loc_2d: Optional[torch.Tensor] = None,
commit_lens: Optional[torch.Tensor] = None,
extend_lens: Optional[torch.Tensor] = None,
) -> None:
"""Materialize target context features into the draft KV cache at explicit slots.
@@ -1835,6 +1863,13 @@ class DFlashWorkerV2(BaseSpecWorker):
),
):
ctx_hidden = self.draft_model.project_target_hidden(target_hidden)
if self.lilicorr is not None:
self._lilicorr_anchor = publish_anchor(
draft_sampler=self._draft_sampler,
ctx_hidden=ctx_hidden,
extend_lens=extend_lens,
commit_lens=commit_lens,
)
if cache_loc_2d is not None:
bs = int(commit_lens.shape[0])
@@ -2186,6 +2221,10 @@ class DFlashWorkerV2(BaseSpecWorker):
self._warned_sampling_fallback = True
return
# The sampled head hands verify the q it drew from, so no fallback is needed.
if self.lilicorr is not None and self._lilicorr_sampling_enabled:
return
if (
not is_dflash_sampling_verify_available()
and not self._warned_sampling_fallback
@@ -2283,6 +2322,7 @@ class DFlashWorkerV2(BaseSpecWorker):
target_hidden=logits_output.hidden_states,
cache_loc=batch.out_cache_loc,
positions=positions,
extend_lens=ctx_lens,
)
# Avoid copying large hidden-state buffers to CPU in overlap scheduling.
@@ -2517,7 +2557,7 @@ class DFlashWorkerV2(BaseSpecWorker):
global_num_token_non_padded_cpu=bs * block_size,
)
if self.selector is not None:
if self.selector is not None or self.lilicorr is not None:
self._selector_sample = None
if self._draft_sampler is not None:
# Consumed by the in-graph sample; must be staged before the replay.
@@ -2588,6 +2628,15 @@ class DFlashWorkerV2(BaseSpecWorker):
self._draft_sampler.candidate_out[:bs],
self._draft_sampler.q_out[:bs],
)
elif (
self.lilicorr is not None
and self._lilicorr_sampling_enabled
and not _is_all_greedy(batch.sampling_info)
):
self._selector_sample = (
self._draft_sampler.candidate_out[:bs],
self._draft_sampler.q_out[:bs],
)
elif self.selector is not None:
with self.draft_tp_context(
self.draft_model_runner.tp_group,
@@ -2600,6 +2649,40 @@ class DFlashWorkerV2(BaseSpecWorker):
anchor_token_ids=block_ids[:, 0],
sampling_info=batch.sampling_info,
)
elif self.lilicorr is not None:
if (
self._lilicorr_sampling_enabled
and not self._warned_lilicorr_eager
and self.model_runner.tp_rank == 0
):
logger.warning(
"LiLiCorr sampled draft ran the eager head on a decode step "
"(draft cuda graph unavailable for batch size %d; captured "
"buckets do not cover it, or graphs are disabled). Acceptance is "
"unaffected but expect a large throughput regression, and do not "
"compare tokens/s from this run against a folded one.",
bs,
)
self._warned_lilicorr_eager = True
draft_hidden = draft_logits_output.hidden_states
if draft_hidden is None:
raise RuntimeError("DFLASH draft model returned no hidden states.")
with self.draft_tp_context(self.draft_model_runner.tp_group):
draft_next, lilicorr_candidate_ids, lilicorr_q_rows = (
propose_lilicorr_block(
head=self.lilicorr,
draft_hidden=draft_hidden.view(bs, int(self.block_size), -1),
lm_head=lm_head,
embed_tokens=target_input_embeddings(
self.target_worker.model_runner.model
),
anchor=self._lilicorr_anchor,
sampling_info=batch.sampling_info,
sampling_enabled=self._lilicorr_sampling_enabled,
)
)
if lilicorr_q_rows is not None and not _is_all_greedy(batch.sampling_info):
self._selector_sample = (lilicorr_candidate_ids, lilicorr_q_rows)
else:
draft_hidden = draft_logits_output.hidden_states
if draft_hidden is None:
@@ -0,0 +1,416 @@
from __future__ import annotations
import logging
from typing import Any, Optional, Tuple
import msgspec
import torch
from sglang.kernels.ops.speculative.lilicorr import MAX_FUSED_CANDIDATE_TOPK
from sglang.srt.environ import envs
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
from sglang.srt.models.dflash import candidate_topk
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.speculative.dflash_utils import _get_dflash_config
from sglang.srt.speculative.dspark_components.dspark_draft import resolve_greedy_mask
logger = logging.getLogger(__name__)
def resolve_sampling_enabled(*, device_supported: bool) -> bool:
requested = envs.SGLANG_ENABLE_LILICORR_SAMPLING.get()
if envs.SGLANG_LILICORR_REQUIRE_SAMPLING.get() and not requested:
raise RuntimeError(
"SGLANG_LILICORR_REQUIRE_SAMPLING is set but SGLANG_ENABLE_LILICORR_SAMPLING "
"is not; this process would serve the greedy commit under a sampled arm's name."
)
logger.info("LiLiCorr draft commit: %s", "sampled" if requested else "greedy")
return requested and device_supported
class LiLiCorrConfig(msgspec.Struct, frozen=True):
candidate_topk: int
hidden_size: int
num_layers: int
num_heads: int
mlp_ratio: float
factor_dim: int
vector_eps: float
logit_scale: float
def resolve_hidden_size(self, *, model_hidden_size: int) -> int:
# 0 means "as wide as the draft", recorded for a head with no token_proj.
return int(self.hidden_size) if self.hidden_size else int(model_hidden_size)
def _parse_lilicorr_config(dflash_cfg: dict) -> Optional[LiLiCorrConfig]:
if not any(key.startswith("lilicorr_") for key in dflash_cfg):
return None
enabled = dflash_cfg.get("lilicorr_enabled")
if enabled is not None and not bool(enabled):
return None
def required(key: str, cast, *, positive: bool = True):
# No defaults: logit_scale and vector_eps change no shape, so load cannot catch them.
full_key = f"lilicorr_{key}"
if full_key not in dflash_cfg:
raise ValueError(
f"DFLASH dflash_config.{full_key} is required to rebuild the LiLiCorr "
"head, and the checkpoint does not carry it."
)
try:
value = cast(dflash_cfg[full_key])
except Exception as e:
raise ValueError(
f"Invalid dflash_config.{full_key}={dflash_cfg[full_key]!r}."
) from e
if positive and value <= 0:
raise ValueError(f"dflash_config.{full_key} must be positive, got {value}.")
return value
candidate_topk = required("candidate_topk", int)
if candidate_topk & (candidate_topk - 1):
raise ValueError(
f"dflash_config.lilicorr_candidate_topk must be a power of two, got "
f"{candidate_topk}: tl.arange requires a power-of-two extent."
)
if candidate_topk > MAX_FUSED_CANDIDATE_TOPK:
raise ValueError(
f"dflash_config.lilicorr_candidate_topk={candidate_topk} exceeds the fused "
f"greedy commit's width of {MAX_FUSED_CANDIDATE_TOPK}, one Triton lane "
"group; a wider head would serve on the reference path."
)
return LiLiCorrConfig(
candidate_topk=candidate_topk,
hidden_size=required("hidden_size", int, positive=False),
num_layers=required("num_layers", int),
num_heads=required("num_heads", int),
mlp_ratio=required("mlp_ratio", float),
factor_dim=required("factor_dim", int),
vector_eps=required("vector_eps", float),
logit_scale=required("logit_scale", float),
)
def parse_lilicorr_draft_config(*, draft_hf_config: Any) -> LiLiCorrConfig:
config = _parse_lilicorr_config(_get_dflash_config(draft_hf_config))
if config is None:
raise ValueError(
"LiLiCorr requires the lilicorr_* geometry fields in dflash_config; a "
'checkpoint declaring architectures=["LiLiCorrDraftModel"] without them '
"cannot be rebuilt."
)
return config
def reject_added_vocab(lm_head) -> None:
if not isinstance(lm_head, VocabParallelEmbedding):
return
if int(lm_head.shard_indices.num_added_elements) != 0:
raise NotImplementedError(
"LiLiCorr's candidate head does not support added vocabulary: those rows "
"sit past the padded base shard, so a contiguous top-k would skip them."
)
def target_input_embeddings(target_model):
# Not the worker's _resolve_dflash_embedding_module, which returns the draft's own
# table for Nemotron-3.5: the head must embed with the table it was trained against.
embed = target_model.get_input_embeddings()
if embed is None:
raise RuntimeError("DFLASH target model exposes no input embeddings.")
return embed
def lilicorr_candidates(
*, hidden_states: torch.Tensor, lm_head, topk: int
) -> Tuple[torch.Tensor, torch.Tensor]:
# log_softmax(logits).topk(topk) over the FULL vocabulary: the head was trained on it.
ids, vals, lse = candidate_topk(
hidden_states, lm_head, int(topk), with_partition=True
)
return vals.float() - lse.unsqueeze(-1), ids
def per_request_last_row(
*,
num_rows: int,
extend_lens: Optional[torch.Tensor],
commit_lens: Optional[torch.Tensor],
) -> Optional[torch.Tensor]:
# Verify's commit_lens index a [bs, block_size]-padded buffer (constant stride);
# prefill/extend's extend_lens index a packed ragged one.
if commit_lens is not None:
bs = int(commit_lens.shape[0])
if bs == 0 or num_rows % bs != 0:
return None
stride = num_rows // bs
base = torch.arange(bs, device=commit_lens.device, dtype=torch.int64) * stride
return (base + commit_lens.to(torch.int64) - 1).clamp_min(0)
if extend_lens is None or extend_lens.numel() == 0:
return None
lens = extend_lens.to(torch.int64).flatten()
return (torch.cumsum(lens, dim=0) - 1).clamp_min(0)
def publish_anchor(
*,
draft_sampler,
ctx_hidden: torch.Tensor,
extend_lens: Optional[torch.Tensor] = None,
commit_lens: Optional[torch.Tensor] = None,
) -> Optional[torch.Tensor]:
ends = per_request_last_row(
num_rows=int(ctx_hidden.shape[0]),
extend_lens=extend_lens,
commit_lens=commit_lens,
)
anchor = None if ends is None else ctx_hidden.index_select(0, ends)
if draft_sampler is not None:
draft_sampler.set_anchor(anchor, 0 if anchor is None else int(anchor.shape[0]))
return anchor
def propose_lilicorr_block(
*,
head,
draft_hidden: torch.Tensor,
lm_head,
embed_tokens,
anchor: Optional[torch.Tensor],
sampling_info=None,
sampling_enabled: bool = False,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]:
bs, block_size, hidden_size = draft_hidden.shape
slots = block_size - 1
pass_hidden = draft_hidden[:, 1:, :]
log_probs, candidate_tokens = lilicorr_candidates(
hidden_states=pass_hidden.reshape(bs * slots, hidden_size),
lm_head=lm_head,
topk=int(head.candidate_topk),
)
candidate_tokens = candidate_tokens.view(bs, slots, int(head.candidate_topk))
feat = int(head.context_proj.in_features)
if anchor is None or int(anchor.shape[0]) != bs:
# The batch resized since the anchor write; score invalid rather than misaligned.
anchor_hidden = torch.zeros(
(bs, feat), device=draft_hidden.device, dtype=draft_hidden.dtype
)
anchor_valid = torch.zeros((bs,), dtype=torch.bool, device=draft_hidden.device)
else:
anchor_hidden = anchor.view(bs, feat)
anchor_valid = torch.ones((bs,), dtype=torch.bool, device=draft_hidden.device)
# Outside the head because on the target model it may be a TP-sharded collective.
common = dict(
token_embeddings=embed_tokens(candidate_tokens).detach(),
candidate_tokens=candidate_tokens,
candidate_log_probs=log_probs.view(bs, slots, int(head.candidate_topk)),
pass_hidden=pass_hidden,
anchor_hidden=anchor_hidden,
anchor_valid=anchor_valid,
)
device = draft_hidden.device
if sampling_enabled and sampling_info is not None:
# A greedy row's temperature may be 0, and it divides before greedy_mask applies.
temperatures = sampling_info.temperatures.view(-1)[:bs].float().clamp_min(1e-5)
else:
temperatures = torch.ones(bs, dtype=torch.float32, device=device)
greedy_mask = (
resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device)
if sampling_enabled
else torch.ones(bs, dtype=torch.bool, device=device)
)
selected, q_rows = head.select_with_proposal(
uniforms=torch.rand(bs, slots, dtype=torch.float32, device=device),
temperatures=temperatures,
greedy_mask=greedy_mask,
**common,
)
if not sampling_enabled:
return selected.to(torch.long), None, None
return selected.to(torch.long), candidate_tokens, q_rows
class LiLiCorrDraftSampler:
def __init__(
self,
*,
head,
embed_tokens,
lm_head,
block_size: int,
max_bs: int,
anchor_features: int,
sampling_enabled: bool,
) -> None:
self.head = head
self.sampling_enabled = bool(sampling_enabled)
self.embed_tokens = embed_tokens
self.lm_head = lm_head
self.block_size = int(block_size)
self.slots = self.block_size - 1
self.topk = int(head.candidate_topk)
self.max_bs = int(max_bs)
# From the head, not from lm_head.weight: a quantized head's weight is packed,
# so its dtype is not the dtype these buffers carry.
anchor_row = next(head.parameters())
device, dtype = anchor_row.device, anchor_row.dtype
max_rows = self.max_bs * self.slots
self.out = torch.empty((max_rows,), dtype=torch.int64, device=device)
self.anchor = torch.zeros(
(self.max_bs, int(anchor_features)), dtype=dtype, device=device
)
self.anchor_valid = torch.zeros((self.max_bs,), dtype=torch.bool, device=device)
self.token_table = head.build_token_table(embed_tokens)
self.temperatures = torch.ones(
(self.max_bs,), dtype=torch.float32, device=device
)
self.greedy_mask = torch.ones((self.max_bs,), dtype=torch.bool, device=device)
self.uniforms = torch.zeros(
(self.max_bs, self.slots), dtype=torch.float32, device=device
)
self.q_out = torch.empty(
(self.max_bs, self.slots, self.topk), dtype=torch.float32, device=device
)
self.candidate_out = torch.empty(
(self.max_bs, self.slots, self.topk), dtype=torch.int64, device=device
)
def set_anchor(self, rows: Optional[torch.Tensor], bs: int) -> None:
# Rows past bs are zeroed, not left stale: the graph runs at the bucket size.
# Filter/merge can hand a row a neighbour's anchor, costing acceptance only.
count = int(bs)
if rows is None or int(rows.shape[0]) != count or count > self.max_bs:
count = 0
if count:
self.anchor[:count].copy_(rows)
self.anchor_valid[:count].fill_(True)
if count < self.max_bs:
self.anchor[count:].zero_()
self.anchor_valid[count:].fill_(False)
def stage_sampling_params(self, *, bs: int, sampling_info) -> None:
if not self.sampling_enabled:
return
if sampling_info is None:
self.temperatures[:bs].fill_(1.0)
self.greedy_mask[:bs].fill_(True)
return
torch.clamp(
sampling_info.temperatures.view(-1)[:bs].to(torch.float32),
min=1e-5,
out=self.temperatures[:bs],
)
self.greedy_mask[:bs].copy_(
resolve_greedy_mask(
bs=bs, sampling_info=sampling_info, device=self.greedy_mask.device
)
)
def __call__(self, hidden_states: torch.Tensor, input_ids=None) -> None:
del input_ids
bs = hidden_states.shape[0] // self.block_size
rows = bs * self.slots
if bs > self.max_bs:
raise RuntimeError(
f"LiLiCorrDraftSampler was built for max_bs={self.max_bs} but the draft "
f"graph replayed at bs={bs}; its static buffers are undersized."
)
hidden_size = hidden_states.shape[-1]
pass_hidden = hidden_states.view(bs, self.block_size, hidden_size)[:, 1:, :]
log_probs, candidate_tokens = lilicorr_candidates(
hidden_states=pass_hidden.reshape(rows, hidden_size),
lm_head=self.lm_head,
topk=self.topk,
)
candidate_tokens = candidate_tokens.view(bs, self.slots, self.topk)
pre_projected = self.token_table is not None
token_embeddings = (
self.token_table[candidate_tokens]
if pre_projected
else self.embed_tokens(candidate_tokens).detach()
)
common = dict(
token_embeddings=token_embeddings,
candidate_tokens=candidate_tokens,
candidate_log_probs=log_probs.view(bs, self.slots, self.topk),
pass_hidden=pass_hidden,
anchor_hidden=self.anchor[:bs],
anchor_valid=self.anchor_valid[:bs],
already_projected=pre_projected,
)
if self.sampling_enabled:
self.uniforms[:bs].uniform_()
selected, q_rows = self.head.select_with_proposal(
uniforms=self.uniforms[:bs],
temperatures=self.temperatures[:bs],
greedy_mask=self.greedy_mask[:bs],
**common,
)
if self.sampling_enabled:
self.q_out[:bs].copy_(q_rows)
self.candidate_out[:bs].copy_(candidate_tokens)
self.out[:rows].copy_(selected.reshape(-1).to(torch.int64))
def draft_graph_batch_sizes() -> list[int]:
return sorted(
{int(bs) for bs in get_exec().graph.cuda_graph_config.decode.bs if bs > 0}
)
def build_lilicorr_draft_sampler(
*,
head,
draft_model,
embed_tokens,
lm_head,
block_size: int,
sampling_enabled: bool = False,
) -> Optional[LiLiCorrDraftSampler]:
def eager(reason: str) -> None:
logger.warning(
"LiLiCorr head kept eager (reason=%s): a bring-up path, not a serving "
"configuration. Expect a large throughput regression and numbers that are "
"not comparable to any published row.",
reason,
)
return None
tp_group = get_parallel().tp_group
if int(tp_group.world_size) != 1:
return eager("tp>1")
batch_sizes = draft_graph_batch_sizes()
if not batch_sizes:
return eager("no draft graph batch sizes to size the static buffers from")
max_bs = batch_sizes[-1]
reject_added_vocab(lm_head)
parameter = next(head.parameters())
head.materialize_inference_buffers(parameter.device, parameter.dtype)
sampler = LiLiCorrDraftSampler(
head=head,
embed_tokens=embed_tokens,
lm_head=lm_head,
block_size=int(block_size),
max_bs=int(max_bs),
anchor_features=int(draft_model.fc.out_features),
sampling_enabled=sampling_enabled,
)
logger.info(
"LiLiCorr select folded into the draft cuda graph: max_bs=%d block_size=%d K=%d.",
max_bs,
int(block_size),
sampler.topk,
)
return sampler