[Spec] Add LiLiCorr: a candidate-lattice reranker for DFlash drafts (#37462)
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: kpham-sgl <khoa.pham@radixark.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Po-Han Huang
kpham-sgl
Claude Opus 5.5
parent
14ab74ef95
commit
78eee88113
@@ -21,6 +21,8 @@ _TRITON_KERNELS = [
|
||||
("topk1", "draft_topk1_postprocess"),
|
||||
("ragged_verify_kernels", "pad_verify_lens_to_bucket"),
|
||||
("ragged_verify_kernels", "build_qo_indptr"),
|
||||
("lilicorr", "lilicorr_topk_lse"),
|
||||
("lilicorr", "lilicorr_sample_path"),
|
||||
("reject_sampling", "chain_speculative_sampling_triton"),
|
||||
("row_argmax", "row_argmax"),
|
||||
("reconstruct_tree", "reconstruct_indices_from_tree_mask_triton"),
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.speculative.dflash import selector_walk_triton
|
||||
|
||||
# Launch geometry tuned on H100 for a ~150k vocabulary; TILE * TPP is the load block.
|
||||
_TILE = 1024
|
||||
_TILES_PER_PROGRAM = 8
|
||||
_NUM_WARPS = 4
|
||||
|
||||
# One Triton lane group in the selector walk; the config refuses a wider head.
|
||||
MAX_FUSED_CANDIDATE_TOPK = 16
|
||||
|
||||
_NEG = -3.0e38
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tiled_scan(
|
||||
lp, # [N, V] logits (any float dtype)
|
||||
tmax, # [N, T] fp32 per-tile max
|
||||
pm, # [N, P] fp32 online-softmax running max
|
||||
ps, # [N, P] fp32 online-softmax running sum
|
||||
V,
|
||||
T,
|
||||
stride_lp_n,
|
||||
stride_tmax_n,
|
||||
stride_p_n,
|
||||
TILE: tl.constexpr,
|
||||
TPP: tl.constexpr,
|
||||
):
|
||||
neg = -3.0e38
|
||||
row = tl.program_id(0)
|
||||
grp = tl.program_id(1)
|
||||
|
||||
t0 = grp * TPP
|
||||
tile_off = t0 + tl.arange(0, TPP)
|
||||
offs = tile_off[:, None] * TILE + tl.arange(0, TILE)[None, :]
|
||||
mask = (tile_off[:, None] < T) & (offs < V)
|
||||
|
||||
x = tl.load(lp + row * stride_lp_n + offs, mask=mask, other=neg)
|
||||
x = x.to(tl.float32)
|
||||
|
||||
tm = tl.max(x, axis=1)
|
||||
tl.store(tmax + row * stride_tmax_n + tile_off, tm, mask=tile_off < T)
|
||||
|
||||
m = tl.max(tm, axis=0)
|
||||
s = tl.sum(tl.exp(x - m), axis=0)
|
||||
s = tl.sum(s, axis=0)
|
||||
tl.store(pm + row * stride_p_n + grp, m)
|
||||
tl.store(ps + row * stride_p_n + grp, s)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tiled_select(
|
||||
lp, # [N, V] logits
|
||||
tids, # [N, KT] int64 candidate tile ids (ascending)
|
||||
ov, # [N, K] fp32 out vals (descending)
|
||||
oi, # [N, K] int64 out ids
|
||||
V,
|
||||
stride_lp_n,
|
||||
stride_tid_n,
|
||||
stride_ov_n,
|
||||
stride_oi_n,
|
||||
TILE: tl.constexpr,
|
||||
KT: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
NSEL: tl.constexpr,
|
||||
):
|
||||
neg = -3.0e38
|
||||
row = tl.program_id(0)
|
||||
|
||||
tid = tl.load(tids + row * stride_tid_n + tl.arange(0, KT))
|
||||
offs2 = tid[:, None] * TILE + tl.arange(0, TILE)[None, :]
|
||||
mask2 = offs2 < V
|
||||
x2 = tl.load(lp + row * stride_lp_n + offs2, mask=mask2, other=neg)
|
||||
|
||||
x = tl.reshape(x2.to(tl.float32), (NSEL,))
|
||||
ids = tl.reshape(offs2, (NSEL,))
|
||||
lane = tl.arange(0, NSEL)
|
||||
|
||||
for j in range(K):
|
||||
mv = tl.max(x, axis=0)
|
||||
mp = tl.argmax(x, axis=0)
|
||||
mid = tl.sum(tl.where(lane == mp, ids, 0), axis=0)
|
||||
tl.store(ov + row * stride_ov_n + j, mv)
|
||||
tl.store(oi + row * stride_oi_n + j, mid)
|
||||
x = tl.where(lane == mp, neg, x)
|
||||
|
||||
|
||||
def _combine_lse(pm: torch.Tensor, ps: torch.Tensor) -> torch.Tensor:
|
||||
gm = pm.max(dim=-1, keepdim=True).values
|
||||
s = (ps * (pm - gm).exp()).sum(dim=-1)
|
||||
return gm.squeeze(-1) + s.log()
|
||||
|
||||
|
||||
def _topk_lse_torch(
|
||||
logits: torch.Tensor, k: int
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
vals, ids = torch.topk(logits, k, dim=-1)
|
||||
rowmax = vals[:, 0:1] # topk is sorted descending
|
||||
sumexp = (logits - rowmax).exp().sum(dim=-1, dtype=torch.float32)
|
||||
lse = rowmax.squeeze(-1).to(torch.float32) + sumexp.log()
|
||||
return vals.float(), ids.to(torch.int64), lse
|
||||
|
||||
|
||||
def lilicorr_topk_lse(
|
||||
logits: torch.Tensor, k: int
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Tile pre-selection is exact: a top-k element's tile has a max at least as large,
|
||||
# so it is among the k largest-max tiles.
|
||||
if not logits.is_cuda:
|
||||
return _topk_lse_torch(logits, k)
|
||||
|
||||
logits = logits.contiguous()
|
||||
n, V = int(logits.shape[0]), int(logits.shape[1])
|
||||
K = int(k)
|
||||
T = (V + _TILE - 1) // _TILE
|
||||
if min(K, T) & (min(K, T) - 1):
|
||||
return _topk_lse_torch(logits, k)
|
||||
P = (T + _TILES_PER_PROGRAM - 1) // _TILES_PER_PROGRAM
|
||||
device = logits.device
|
||||
|
||||
tmax = torch.full((n, T), _NEG, dtype=torch.float32, device=device)
|
||||
pm = torch.full((n, P), _NEG, dtype=torch.float32, device=device)
|
||||
ps = torch.zeros((n, P), dtype=torch.float32, device=device)
|
||||
_tiled_scan[(n, P)](
|
||||
logits,
|
||||
tmax,
|
||||
pm,
|
||||
ps,
|
||||
V,
|
||||
T,
|
||||
logits.stride(0),
|
||||
tmax.stride(0),
|
||||
pm.stride(0),
|
||||
TILE=_TILE,
|
||||
TPP=_TILES_PER_PROGRAM,
|
||||
num_warps=_NUM_WARPS,
|
||||
)
|
||||
|
||||
kt = min(K, T)
|
||||
tids = torch.topk(tmax, kt, dim=-1).indices
|
||||
tids = torch.sort(tids, dim=-1).values.to(torch.int64)
|
||||
|
||||
ov = torch.empty((n, K), dtype=torch.float32, device=device)
|
||||
oi = torch.empty((n, K), dtype=torch.int64, device=device)
|
||||
_tiled_select[(n,)](
|
||||
logits,
|
||||
tids,
|
||||
ov,
|
||||
oi,
|
||||
V,
|
||||
logits.stride(0),
|
||||
tids.stride(0),
|
||||
ov.stride(0),
|
||||
oi.stride(0),
|
||||
TILE=_TILE,
|
||||
KT=kt,
|
||||
K=K,
|
||||
NSEL=kt * _TILE,
|
||||
num_warps=_NUM_WARPS,
|
||||
)
|
||||
return ov, oi, _combine_lse(pm, ps)
|
||||
|
||||
|
||||
def _lattice_scores(log_start: torch.Tensor, log_pair: torch.Tensor) -> torch.Tensor:
|
||||
# The walk reads only scores[:, 0, 0, :] at slot 0, so broadcasting is sound.
|
||||
topk = int(log_start.shape[-1])
|
||||
start = log_start.float()[:, None, None, :].expand(-1, 1, topk, topk)
|
||||
return torch.cat([start, log_pair.float()], dim=1)
|
||||
|
||||
|
||||
def _selector_walk_torch(
|
||||
*,
|
||||
candidate_ids: torch.Tensor,
|
||||
scores: torch.Tensor,
|
||||
uniforms: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
greedy_mask: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
_, num_slots, topk = candidate_ids.shape
|
||||
temps = temperatures.view(-1, 1).to(torch.float32)
|
||||
greedy = greedy_mask.view(-1)
|
||||
previous = torch.zeros(greedy.shape[0], dtype=torch.int64, device=scores.device)
|
||||
tokens, q_rows = [], []
|
||||
for slot in range(num_slots):
|
||||
node = torch.gather(
|
||||
scores[:, slot].float(), 1, previous.view(-1, 1, 1).expand(-1, 1, topk)
|
||||
).squeeze(1)
|
||||
probs = torch.softmax(node / temps, dim=-1)
|
||||
sampled = (
|
||||
uniforms[:, slot : slot + 1]
|
||||
.ge(probs.cumsum(dim=-1))
|
||||
.sum(dim=-1)
|
||||
.clamp_max(topk - 1)
|
||||
)
|
||||
previous = torch.where(greedy, node.argmax(dim=-1), sampled)
|
||||
q_rows.append(
|
||||
torch.where(
|
||||
greedy.unsqueeze(-1),
|
||||
F.one_hot(previous, topk).to(torch.float32),
|
||||
probs,
|
||||
)
|
||||
)
|
||||
tokens.append(
|
||||
torch.gather(candidate_ids[:, slot], 1, previous.view(-1, 1)).squeeze(1)
|
||||
)
|
||||
return torch.stack(tokens, dim=-1).to(torch.int64), torch.stack(q_rows, dim=1)
|
||||
|
||||
|
||||
def lilicorr_sample_path(
|
||||
log_start: torch.Tensor,
|
||||
log_pair: torch.Tensor,
|
||||
candidate_tokens: torch.Tensor,
|
||||
uniforms: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
greedy_mask: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
# No log-prob prior in the proposal: the head was trained without one. Greedy rows
|
||||
# report a point mass so min(1, p/q) stays the right acceptance test.
|
||||
scores = _lattice_scores(log_start, log_pair)
|
||||
walk = selector_walk_triton if scores.is_cuda else _selector_walk_torch
|
||||
return walk(
|
||||
candidate_ids=candidate_tokens,
|
||||
scores=scores,
|
||||
uniforms=uniforms,
|
||||
temperatures=temperatures,
|
||||
greedy_mask=greedy_mask,
|
||||
)
|
||||
@@ -1366,6 +1366,8 @@ class Envs:
|
||||
SGLANG_OPT_FUSED_KDA_VERIFY = EnvBool(False)
|
||||
# A/B: keep the DFLASH draft greedy head eager (not folded in-graph).
|
||||
SGLANG_DFLASH_EAGER_DRAFT_SAMPLER = EnvBool(False)
|
||||
SGLANG_ENABLE_LILICORR_SAMPLING = EnvBool(False)
|
||||
SGLANG_LILICORR_REQUIRE_SAMPLING = EnvBool(False)
|
||||
SGLANG_RAGGED_VERIFY_MODE = EnvStr("static")
|
||||
SGLANG_TEST_RAGGED_VERIFY_FORCE_UNIFORM_CAPTURE = EnvBool(False)
|
||||
# Skip draft_extend while adaptive spec is at steps=0 (drafting disabled).
|
||||
|
||||
@@ -12,6 +12,7 @@ import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.speculative.dflash import selector_walk_triton
|
||||
from sglang.kernels.ops.speculative.lilicorr import lilicorr_topk_lse
|
||||
from sglang.srt.configs.laguna import normalize_gating
|
||||
from sglang.srt.distributed.communication_op import tensor_model_parallel_all_gather
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
@@ -101,6 +102,56 @@ def _project_candidate_logits(
|
||||
return logits
|
||||
|
||||
|
||||
def candidate_topk(
|
||||
hidden: torch.Tensor,
|
||||
lm_head: nn.Module,
|
||||
k: int,
|
||||
*,
|
||||
with_partition: bool = False,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Global top-k candidates from the target head: hidden [N, H] -> ids / vals [N, K].
|
||||
|
||||
Under TP (vocab-sharded lm_head): local top-k per shard, all-gather K logits/ids (not
|
||||
the full vocab), then a global top-k -- identical candidates at O(tp*K) instead of
|
||||
O(vocab) gather bandwidth. with_partition also returns the full-vocabulary lse [N] fp32.
|
||||
"""
|
||||
# The worker screens the head before capture, but the eager fallbacks attach
|
||||
# whatever the target has.
|
||||
weight = getattr(lm_head, "weight", None)
|
||||
quant_method = getattr(lm_head, "quant_method", None)
|
||||
use_quant_head = should_apply_lm_head_quant_method(lm_head, quant_method)
|
||||
if not use_quant_head and not is_dense_head_weight(weight):
|
||||
raise RuntimeError(
|
||||
"DFlash candidate top-k requires a dense FP16/BF16/FP32 target lm_head "
|
||||
"or a supported lm_head.quant_method."
|
||||
)
|
||||
tp = get_parallel().tp_size == 1
|
||||
num_org = (
|
||||
int(lm_head.org_vocab_size)
|
||||
if tp
|
||||
else int(lm_head.shard_indices.num_org_elements)
|
||||
)
|
||||
logits = _project_candidate_logits(
|
||||
hidden, lm_head, num_org=num_org, use_quant_head=use_quant_head
|
||||
)
|
||||
lse = None
|
||||
if with_partition:
|
||||
vals, ids, lse = lilicorr_topk_lse(logits, k)
|
||||
else:
|
||||
vals, ids = _radix_topk(logits, k)
|
||||
if tp:
|
||||
return ids.long(), vals, lse
|
||||
global_ids = ids.long() + int(lm_head.shard_indices.org_vocab_start_index)
|
||||
gathered_vals = tensor_model_parallel_all_gather(vals.float(), dim=-1)
|
||||
gathered_ids = tensor_model_parallel_all_gather(global_ids, dim=-1)
|
||||
top_vals, sel = torch.topk(gathered_vals, k, dim=-1)
|
||||
if lse is not None:
|
||||
lse = torch.logsumexp(
|
||||
tensor_model_parallel_all_gather(lse.unsqueeze(-1), dim=-1), dim=-1
|
||||
)
|
||||
return torch.gather(gathered_ids, -1, sel).long(), top_vals, lse
|
||||
|
||||
|
||||
def _get_dflash_attention_type(config, *, default: AttentionType) -> AttentionType:
|
||||
"""Honor explicit causality while preserving legacy layer defaults."""
|
||||
text_config = config.get_text_config()
|
||||
@@ -579,11 +630,13 @@ class DFlashDraftModel(nn.Module):
|
||||
hidden_size = int(config.hidden_size)
|
||||
num_layers = int(config.num_hidden_layers)
|
||||
rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-6))
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
draft_config = self.draft_config = parse_dflash_draft_config(
|
||||
draft_hf_config=config
|
||||
)
|
||||
self.block_size = draft_config.resolve_block_size(default=16)
|
||||
self.candidate_selector: Optional[nn.Module] = None
|
||||
self.lilicorr: Optional[nn.Module] = None
|
||||
self.is_nemotron_35_draft = is_nemotron_35_draft_config(config)
|
||||
self.embed_tokens: Optional[VocabParallelEmbedding] = None
|
||||
if self.is_nemotron_35_draft:
|
||||
@@ -1176,47 +1229,12 @@ class DFlash2DraftModel(DFlashDraftModel):
|
||||
self, hidden: torch.Tensor
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Top-k base candidates via the target lm_head: hidden [N, H] -> global
|
||||
candidate_ids / unary_logits [N, K]. Under TP (vocab-sharded lm_head): local top-k
|
||||
per shard, all-gather K logits/ids (not the full vocab), then a global top-k --
|
||||
identical candidates at O(tp*K) instead of O(vocab) gather bandwidth."""
|
||||
candidate_ids / unary_logits [N, K]."""
|
||||
assert self.lm_head is not None, "draft_model.lm_head unset before capture"
|
||||
k = self.candidate_selector.top_k
|
||||
# The worker screens the head before capture, but its eager fallback
|
||||
# (_propose_selector_block) attaches whatever the target has.
|
||||
weight = getattr(self.lm_head, "weight", None)
|
||||
quant_method = getattr(self.lm_head, "quant_method", None)
|
||||
use_quant_head = should_apply_lm_head_quant_method(self.lm_head, quant_method)
|
||||
if not use_quant_head and not is_dense_head_weight(weight):
|
||||
raise RuntimeError(
|
||||
"DFlash2 selector requires a dense FP16/BF16/FP32 target lm_head "
|
||||
"or a supported lm_head.quant_method."
|
||||
)
|
||||
if get_parallel().tp_size == 1:
|
||||
org = int(self.lm_head.org_vocab_size)
|
||||
vals, ids = _radix_topk(
|
||||
_project_candidate_logits(
|
||||
hidden, self.lm_head, num_org=org, use_quant_head=use_quant_head
|
||||
),
|
||||
k,
|
||||
)
|
||||
return ids.long(), self._transform_unary_logits(vals)
|
||||
shard = self.lm_head.shard_indices
|
||||
vals, ids = _radix_topk(
|
||||
_project_candidate_logits(
|
||||
hidden,
|
||||
self.lm_head,
|
||||
num_org=int(shard.num_org_elements),
|
||||
use_quant_head=use_quant_head,
|
||||
),
|
||||
k,
|
||||
)
|
||||
global_ids = ids.long() + int(shard.org_vocab_start_index)
|
||||
gathered_vals = tensor_model_parallel_all_gather(vals.float(), dim=-1)
|
||||
gathered_ids = tensor_model_parallel_all_gather(global_ids, dim=-1)
|
||||
top_vals, sel = torch.topk(gathered_vals, k, dim=-1)
|
||||
return torch.gather(gathered_ids, -1, sel).long(), self._transform_unary_logits(
|
||||
top_vals
|
||||
ids, vals, _ = candidate_topk(
|
||||
hidden, self.lm_head, self.candidate_selector.top_k
|
||||
)
|
||||
return ids, self._transform_unary_logits(vals)
|
||||
|
||||
|
||||
class MuseGlimmerAssistantModel(DFlashDraftModel):
|
||||
|
||||
@@ -0,0 +1,473 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Iterable, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.speculative.lilicorr import lilicorr_sample_path
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.models.dflash import DFlashDraftModel
|
||||
from sglang.srt.speculative.lilicorr_utils import (
|
||||
LiLiCorrConfig,
|
||||
parse_lilicorr_draft_config,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LiLiCorrLatticeAttention(nn.Module):
|
||||
# nn.MultiheadAttention's parameter layout, run via SDPA since that module is not capturable.
|
||||
|
||||
def __init__(self, hidden_size: int, num_heads: int) -> None:
|
||||
super().__init__()
|
||||
if hidden_size % num_heads != 0:
|
||||
raise ValueError(
|
||||
f"LiLiCorr hidden_size={hidden_size} must be divisible by "
|
||||
f"num_heads={num_heads}."
|
||||
)
|
||||
self.hidden_size = int(hidden_size)
|
||||
self.num_heads = int(num_heads)
|
||||
self.head_dim = self.hidden_size // self.num_heads
|
||||
self.in_proj_weight = nn.Parameter(torch.zeros(3 * hidden_size, hidden_size))
|
||||
self.in_proj_bias = nn.Parameter(torch.zeros(3 * hidden_size))
|
||||
self.out_proj = nn.Linear(hidden_size, hidden_size)
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, attention_bias: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
bsz, seq_len, _ = hidden_states.shape
|
||||
qkv = F.linear(hidden_states, self.in_proj_weight, self.in_proj_bias)
|
||||
q, k, v = qkv.chunk(3, dim=-1)
|
||||
shape = (bsz, seq_len, self.num_heads, self.head_dim)
|
||||
q = q.view(shape).transpose(1, 2)
|
||||
k = k.view(shape).transpose(1, 2)
|
||||
v = v.view(shape).transpose(1, 2)
|
||||
# attention_bias is [1, heads, L, L]; SDPA broadcasts it over the batch.
|
||||
out = F.scaled_dot_product_attention(q, k, v, attn_mask=attention_bias)
|
||||
return self.out_proj(
|
||||
out.transpose(1, 2).reshape(bsz, seq_len, self.hidden_size)
|
||||
)
|
||||
|
||||
|
||||
class LiLiCorrLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
mlp_ratio: float,
|
||||
rms_norm_eps: float,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.attn_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
|
||||
self.attn = LiLiCorrLatticeAttention(hidden_size, num_heads)
|
||||
self.mlp_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
|
||||
mlp_hidden_size = int(hidden_size * mlp_ratio)
|
||||
self.mlp = nn.Sequential(
|
||||
nn.Linear(hidden_size, mlp_hidden_size),
|
||||
nn.SiLU(),
|
||||
nn.Linear(mlp_hidden_size, hidden_size),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, attention_bias: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
hidden_states = hidden_states + self.attn(
|
||||
self.attn_norm(hidden_states), attention_bias
|
||||
)
|
||||
return hidden_states + self.mlp(self.mlp_norm(hidden_states))
|
||||
|
||||
|
||||
class LiLiCorrHead(nn.Module):
|
||||
# [log_probs, probs, logprob_gap, rank_frac, is_top1], in the trained Linear's order.
|
||||
num_candidate_features = 5
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
model_hidden_size: int,
|
||||
block_size: int,
|
||||
rms_norm_eps: float,
|
||||
config: LiLiCorrConfig,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
hidden_size = config.resolve_hidden_size(model_hidden_size=model_hidden_size)
|
||||
self.block_size = int(block_size)
|
||||
self.num_candidate_slots = self.block_size - 1
|
||||
self.num_active_slots = self.num_candidate_slots
|
||||
self.candidate_topk = int(config.candidate_topk)
|
||||
self.hidden_size = hidden_size
|
||||
self.num_heads = int(config.num_heads)
|
||||
self.mlp_ratio = float(config.mlp_ratio)
|
||||
self.factor_dim = int(config.factor_dim)
|
||||
self.vector_eps = float(config.vector_eps)
|
||||
self.logit_scale = float(config.logit_scale)
|
||||
|
||||
self.token_proj = (
|
||||
nn.Identity()
|
||||
if model_hidden_size == hidden_size
|
||||
else nn.Linear(model_hidden_size, hidden_size)
|
||||
)
|
||||
self.pass_hidden_proj = nn.Linear(model_hidden_size, hidden_size)
|
||||
self.feature_mlp = nn.Sequential(
|
||||
nn.LayerNorm(self.num_candidate_features),
|
||||
nn.Linear(self.num_candidate_features, hidden_size),
|
||||
nn.SiLU(),
|
||||
nn.Linear(hidden_size, hidden_size),
|
||||
)
|
||||
self.slot_embedding = nn.Parameter(
|
||||
torch.zeros(1, 1, self.num_candidate_slots, 1, hidden_size)
|
||||
)
|
||||
self.rank_embedding = nn.Parameter(
|
||||
torch.zeros(1, 1, 1, self.candidate_topk, hidden_size)
|
||||
)
|
||||
self.relative_slot_bias = nn.Parameter(
|
||||
torch.zeros(self.num_heads, 2 * self.block_size - 1)
|
||||
)
|
||||
self.same_slot_bias = nn.Parameter(torch.zeros(self.num_heads))
|
||||
self.context_proj = nn.Linear(model_hidden_size, hidden_size)
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
LiLiCorrLayer(
|
||||
hidden_size=hidden_size,
|
||||
num_heads=self.num_heads,
|
||||
mlp_ratio=self.mlp_ratio,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
)
|
||||
for _ in range(int(config.num_layers))
|
||||
]
|
||||
)
|
||||
self.output_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
|
||||
self.anchor_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
|
||||
self.factor_input_proj = nn.Linear(hidden_size * 3, hidden_size)
|
||||
self.out_head = nn.Linear(hidden_size, self.factor_dim)
|
||||
self.in_head = nn.Linear(hidden_size, self.factor_dim)
|
||||
self.anchor_out_head = nn.Linear(hidden_size, self.factor_dim)
|
||||
|
||||
self._attn_bias: Optional[torch.Tensor] = None
|
||||
self._fused_edge_weight: Optional[torch.Tensor] = None
|
||||
self._fused_edge_bias: Optional[torch.Tensor] = None
|
||||
self._factor_input_splits: Optional[Tuple[torch.Tensor, ...]] = None
|
||||
self._rank_frac_col: Optional[torch.Tensor] = None
|
||||
self._is_top1_col: Optional[torch.Tensor] = None
|
||||
|
||||
@torch.no_grad()
|
||||
def materialize_inference_buffers(
|
||||
self, device: torch.device, dtype: torch.dtype
|
||||
) -> None:
|
||||
# After weight load and before graph capture: host work, not capturable.
|
||||
topk = self.candidate_topk
|
||||
self._attn_bias = self._build_attention_bias(device=device, dtype=dtype)
|
||||
|
||||
self._fused_edge_weight = (
|
||||
torch.cat([self.out_head.weight, self.in_head.weight], dim=0)
|
||||
.to(device=device, dtype=dtype)
|
||||
.contiguous()
|
||||
)
|
||||
self._fused_edge_bias = (
|
||||
torch.cat([self.out_head.bias, self.in_head.bias], dim=0)
|
||||
.to(device=device, dtype=dtype)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
weight = self.factor_input_proj.weight
|
||||
hdim = self.hidden_size
|
||||
self._factor_input_splits = (
|
||||
weight[:, :hdim].contiguous(),
|
||||
weight[:, hdim : 2 * hdim].contiguous(),
|
||||
weight[:, 2 * hdim :].contiguous(),
|
||||
)
|
||||
|
||||
if topk > 1:
|
||||
rank_frac = torch.arange(topk, device=device, dtype=torch.float32).view(
|
||||
1, 1, 1, topk
|
||||
) / float(topk - 1)
|
||||
else:
|
||||
rank_frac = torch.zeros(1, 1, 1, topk, device=device, dtype=torch.float32)
|
||||
is_top1 = torch.zeros(1, 1, 1, topk, device=device, dtype=torch.float32)
|
||||
is_top1[..., 0] = 1.0
|
||||
self._rank_frac_col = rank_frac.contiguous()
|
||||
self._is_top1_col = is_top1.contiguous()
|
||||
|
||||
def _build_attention_bias(
|
||||
self, *, device: torch.device, dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
topk = self.candidate_topk
|
||||
slot_ids = torch.arange(
|
||||
self.num_active_slots, device=device, dtype=torch.long
|
||||
).repeat_interleave(topk)
|
||||
rel = slot_ids.view(-1, 1) - slot_ids.view(1, -1)
|
||||
rel = rel.clamp(min=-(self.block_size - 1), max=self.block_size - 1)
|
||||
bias = self.relative_slot_bias[:, rel + self.block_size - 1]
|
||||
same_slot = slot_ids.view(-1, 1) == slot_ids.view(1, -1)
|
||||
bias = bias + same_slot.unsqueeze(0).to(dtype=bias.dtype) * (
|
||||
self.same_slot_bias.view(-1, 1, 1)
|
||||
)
|
||||
return bias.to(device=device, dtype=dtype).contiguous()
|
||||
|
||||
def set_active_block_size(self, block_size: int) -> None:
|
||||
# A prefix of the trained slots; every relative offset it needs was trained.
|
||||
if not 2 <= int(block_size) <= self.block_size:
|
||||
raise ValueError(
|
||||
f"block_size={int(block_size)} is outside [2, {self.block_size}], "
|
||||
"the trained LiLiCorr block."
|
||||
)
|
||||
self.num_active_slots = int(block_size) - 1
|
||||
if self._attn_bias is not None:
|
||||
self._attn_bias = self._build_attention_bias(
|
||||
device=self._attn_bias.device, dtype=self._attn_bias.dtype
|
||||
)
|
||||
|
||||
def _require_materialized(self) -> None:
|
||||
if self._attn_bias is None:
|
||||
raise RuntimeError(
|
||||
"LiLiCorr head scored before materialize_inference_buffers(): its "
|
||||
"attention bias and fused edge heads are unbuilt."
|
||||
)
|
||||
|
||||
def _project_anchor(
|
||||
self, anchor_hidden: torch.Tensor, anchor_valid: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
# Branch-free so there is no host sync inside the captured region.
|
||||
anchor = self.context_proj(anchor_hidden)
|
||||
return anchor * anchor_valid.unsqueeze(-1).to(anchor.dtype)
|
||||
|
||||
def score(
|
||||
self,
|
||||
*,
|
||||
token_embeddings: torch.Tensor,
|
||||
candidate_log_probs: torch.Tensor,
|
||||
pass_hidden: torch.Tensor,
|
||||
anchor_hidden: torch.Tensor,
|
||||
anchor_valid: torch.Tensor,
|
||||
already_projected: bool = False,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
self._require_materialized()
|
||||
bsz, n_blocks, n_slots, topk = candidate_log_probs.shape
|
||||
if topk != self.candidate_topk:
|
||||
raise ValueError(
|
||||
f"LiLiCorr was built for candidate_topk={self.candidate_topk} but the "
|
||||
f"lattice carries {topk}; both are sized for the trained width."
|
||||
)
|
||||
|
||||
proj_dtype = self.pass_hidden_proj.weight.dtype
|
||||
if token_embeddings.dtype != proj_dtype:
|
||||
token_embeddings = token_embeddings.to(proj_dtype)
|
||||
if pass_hidden.dtype != proj_dtype:
|
||||
pass_hidden = pass_hidden.to(proj_dtype)
|
||||
|
||||
token_states = (
|
||||
token_embeddings if already_projected else self.token_proj(token_embeddings)
|
||||
)
|
||||
pass_states = self.pass_hidden_proj(pass_hidden).unsqueeze(-2)
|
||||
|
||||
log_probs = candidate_log_probs.float()
|
||||
features = torch.stack(
|
||||
[
|
||||
log_probs,
|
||||
log_probs.exp(),
|
||||
log_probs - log_probs.max(dim=-1, keepdim=True).values,
|
||||
self._rank_frac_col.expand_as(log_probs),
|
||||
self._is_top1_col.expand_as(log_probs),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
hidden_states = token_states + pass_states
|
||||
hidden_states = hidden_states + self.feature_mlp(
|
||||
features.to(dtype=token_states.dtype)
|
||||
)
|
||||
hidden_states = hidden_states + self.slot_embedding[:, :, :n_slots]
|
||||
hidden_states = hidden_states + self.rank_embedding
|
||||
hidden_states = hidden_states.reshape(
|
||||
bsz * n_blocks, n_slots * topk, self.hidden_size
|
||||
)
|
||||
|
||||
anchor_state = self._project_anchor(anchor_hidden, anchor_valid)
|
||||
attention_bias = self._attn_bias.unsqueeze(0)
|
||||
for layer in self.layers:
|
||||
hidden_states = layer(hidden_states, attention_bias)
|
||||
hidden_states = self.output_norm(hidden_states).reshape(
|
||||
bsz, n_blocks, n_slots, topk, self.hidden_size
|
||||
)
|
||||
anchor_state = self.anchor_norm(anchor_state)
|
||||
|
||||
w_self, w_anchor, w_cross = self._factor_input_splits
|
||||
anchor_row = anchor_state[:, :, None, None, :]
|
||||
pre = F.linear(hidden_states, w_self, self.factor_input_proj.bias)
|
||||
pre = pre + F.linear(anchor_row, w_anchor)
|
||||
pre = pre + F.linear(hidden_states * anchor_row, w_cross)
|
||||
factor_hidden = F.silu(pre)
|
||||
|
||||
edges = F.linear(factor_hidden, self._fused_edge_weight, self._fused_edge_bias)
|
||||
out_vec, in_vec = F.normalize(
|
||||
edges.unflatten(-1, (2, self.factor_dim)),
|
||||
dim=-1,
|
||||
eps=self.vector_eps,
|
||||
).unbind(-2)
|
||||
anchor_out = F.normalize(
|
||||
self.anchor_out_head(anchor_state), dim=-1, eps=self.vector_eps
|
||||
)
|
||||
|
||||
start_scores = (anchor_out[:, :, None, :] * in_vec[:, :, 0, :, :]).sum(dim=-1)
|
||||
pair_scores = torch.matmul(
|
||||
out_vec[:, :, :-1], in_vec[:, :, 1:].transpose(-1, -2)
|
||||
)
|
||||
return start_scores, pair_scores
|
||||
|
||||
def log_factors(
|
||||
self, start_scores: torch.Tensor, pair_scores: torch.Tensor
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
return (
|
||||
self.logit_scale * start_scores.float(),
|
||||
self.logit_scale * pair_scores.float(),
|
||||
)
|
||||
|
||||
def select_with_proposal(
|
||||
self,
|
||||
*,
|
||||
token_embeddings: torch.Tensor,
|
||||
candidate_tokens: torch.Tensor,
|
||||
candidate_log_probs: torch.Tensor,
|
||||
pass_hidden: torch.Tensor,
|
||||
anchor_hidden: torch.Tensor,
|
||||
anchor_valid: torch.Tensor,
|
||||
uniforms: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
greedy_mask: torch.Tensor,
|
||||
already_projected: bool = False,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
start_scores, pair_scores = self.score(
|
||||
token_embeddings=token_embeddings.unsqueeze(1),
|
||||
candidate_log_probs=candidate_log_probs.unsqueeze(1),
|
||||
pass_hidden=pass_hidden.unsqueeze(1),
|
||||
anchor_hidden=anchor_hidden.unsqueeze(1),
|
||||
anchor_valid=anchor_valid.unsqueeze(1),
|
||||
already_projected=already_projected,
|
||||
)
|
||||
log_start, log_pair = self.log_factors(start_scores, pair_scores)
|
||||
return lilicorr_sample_path(
|
||||
log_start[:, 0, :],
|
||||
log_pair[:, 0],
|
||||
candidate_tokens,
|
||||
uniforms=uniforms,
|
||||
temperatures=temperatures,
|
||||
greedy_mask=greedy_mask,
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def build_token_table(self, embed_tokens: nn.Module) -> Optional[torch.Tensor]:
|
||||
if isinstance(self.token_proj, nn.Identity):
|
||||
return None
|
||||
weight = embed_tokens.weight
|
||||
if int(self.token_proj.weight.shape[1]) != int(weight.shape[1]):
|
||||
return None
|
||||
return F.linear(
|
||||
weight.to(self.token_proj.weight.dtype),
|
||||
self.token_proj.weight,
|
||||
self.token_proj.bias,
|
||||
).contiguous()
|
||||
|
||||
|
||||
def check_head_weight_coverage(head: LiLiCorrHead, seen: set) -> None:
|
||||
# The base loader silently drops unresolved weights, which would only show
|
||||
# up as a lower acceptance length.
|
||||
expected = {f"lilicorr.{name}" for name, _ in head.named_parameters()}
|
||||
missing = sorted(expected - seen)
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"LiLiCorr checkpoint is missing {len(missing)} head parameters "
|
||||
f"(e.g. {missing[:5]}); a draft with no LiLiCorr head should declare "
|
||||
'architectures=["DFlashDraftModel"].'
|
||||
)
|
||||
unexpected = sorted(seen - expected)
|
||||
if unexpected:
|
||||
raise ValueError(
|
||||
f"LiLiCorr checkpoint carries {len(unexpected)} head tensors this head "
|
||||
f"has no parameter for (e.g. {unexpected[:5]}). Check "
|
||||
"lilicorr_hidden_size, lilicorr_num_layers and lilicorr_factor_dim in "
|
||||
"dflash_config against the checkpoint."
|
||||
)
|
||||
|
||||
|
||||
def check_conv_weight_coverage(model: DFlashDraftModel, seen: set) -> None:
|
||||
# conv_kernel_size / conv_group_size default to 0, so a config that lost them
|
||||
# builds no conv modules and the loader drops every conv tensor silently.
|
||||
expected = {
|
||||
name
|
||||
for name, _ in model.named_parameters()
|
||||
if ".attention_conv." in name or ".mlp_conv." in name
|
||||
}
|
||||
|
||||
if seen and not expected:
|
||||
raise ValueError(
|
||||
f"Draft checkpoint carries {len(seen)} grouped-convolution tensors "
|
||||
f"(e.g. {sorted(seen)[:3]}) but this draft built no convolution modules: "
|
||||
"dflash_config is missing conv_kernel_size / conv_group_size, which both "
|
||||
"default to 0 and cannot be inferred from the tensors."
|
||||
)
|
||||
if expected and not seen:
|
||||
raise ValueError(
|
||||
f"This draft built {len(expected)} grouped-convolution parameters from "
|
||||
"dflash_config, but the checkpoint carries none, so kernel_projection "
|
||||
"would serve at its random initialization."
|
||||
)
|
||||
|
||||
missing = sorted(expected - seen)
|
||||
unexpected = sorted(seen - expected)
|
||||
if missing or unexpected:
|
||||
raise ValueError(
|
||||
"Draft checkpoint's grouped-convolution tensors do not correspond to the "
|
||||
f"built ones: {len(missing)} missing (e.g. {missing[:3]}), "
|
||||
f"{len(unexpected)} unexpected (e.g. {unexpected[:3]}). Check "
|
||||
"conv_kernel_size, conv_group_size and num_hidden_layers in dflash_config."
|
||||
)
|
||||
|
||||
if expected:
|
||||
conv = model.layers[0].attention_conv
|
||||
logger.info(
|
||||
"DFLASH grouped convolution live: %d taps, group size %d, %d tensors.",
|
||||
int(conv.taps),
|
||||
int(conv.group_size),
|
||||
len(expected),
|
||||
)
|
||||
|
||||
|
||||
class LiLiCorrDraftModel(DFlashDraftModel):
|
||||
def __init__(self, config, quant_config=None, prefix: str = "") -> None:
|
||||
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
|
||||
self.lilicorr = LiLiCorrHead(
|
||||
model_hidden_size=int(config.hidden_size),
|
||||
block_size=int(self.block_size),
|
||||
rms_norm_eps=self.rms_norm_eps,
|
||||
config=parse_lilicorr_draft_config(draft_hf_config=config),
|
||||
)
|
||||
|
||||
def set_block_size(self, block_size: int) -> None:
|
||||
super().set_block_size(block_size)
|
||||
self.lilicorr.set_active_block_size(block_size)
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
seen: set[str] = set()
|
||||
seen_conv: set[str] = set()
|
||||
|
||||
def tracking():
|
||||
for name, weight in weights:
|
||||
stripped = name[len("model.") :] if name.startswith("model.") else name
|
||||
if stripped.startswith("lilicorr."):
|
||||
seen.add(stripped)
|
||||
elif ".attention_conv." in stripped or ".mlp_conv." in stripped:
|
||||
seen_conv.add(stripped)
|
||||
yield name, weight
|
||||
|
||||
super().load_weights(tracking())
|
||||
|
||||
check_head_weight_coverage(self.lilicorr, seen)
|
||||
check_conv_weight_coverage(self, seen_conv)
|
||||
|
||||
parameter = next(self.lilicorr.parameters())
|
||||
self.lilicorr.materialize_inference_buffers(parameter.device, parameter.dtype)
|
||||
|
||||
|
||||
EntryClass = [LiLiCorrDraftModel]
|
||||
@@ -74,6 +74,13 @@ from sglang.srt.speculative.draft_worker_common import (
|
||||
make_draft_sampler_capture_hook,
|
||||
)
|
||||
from sglang.srt.speculative.dspark_components.dspark_draft import resolve_greedy_mask
|
||||
from sglang.srt.speculative.lilicorr_utils import (
|
||||
build_lilicorr_draft_sampler,
|
||||
propose_lilicorr_block,
|
||||
publish_anchor,
|
||||
resolve_sampling_enabled,
|
||||
target_input_embeddings,
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite
|
||||
from sglang.srt.speculative.spec_utils import (
|
||||
@@ -386,6 +393,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
self.device = target_worker.device
|
||||
|
||||
self._warned_sampling_fallback = False
|
||||
self._warned_lilicorr_eager = False
|
||||
self._draft_probs_buf = None
|
||||
self._logged_first_verify = False
|
||||
self._full_embed_gpu: Optional[torch.Tensor] = None
|
||||
@@ -425,6 +433,12 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
self.selector = self.draft_model.candidate_selector
|
||||
# Ascend keeps selector proposal aligned with its greedy-only verify path.
|
||||
self._selector_sampling_enabled = not _is_npu
|
||||
self.lilicorr = self.draft_model.lilicorr
|
||||
# The sampled commit rides the selector's accept path.
|
||||
self._lilicorr_sampling_enabled = self.lilicorr is not None and (
|
||||
resolve_sampling_enabled(device_supported=self._selector_sampling_enabled)
|
||||
)
|
||||
self._lilicorr_anchor: Optional[torch.Tensor] = None
|
||||
draft_config = parse_dflash_draft_config(
|
||||
draft_hf_config=self.draft_model_runner.model_config.hf_config
|
||||
)
|
||||
@@ -805,15 +819,17 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
if lm_head is None:
|
||||
return _eager("no target lm_head")
|
||||
|
||||
# A gate-admitted quantized head is capture-safe: the target's own
|
||||
# logits path already runs the same kernel under CUDA graphs.
|
||||
head_supported = is_dense_head_weight(
|
||||
getattr(lm_head, "weight", None)
|
||||
) or should_apply_lm_head_quant_method(
|
||||
lm_head, getattr(lm_head, "quant_method", None)
|
||||
)
|
||||
|
||||
if self.selector is not None:
|
||||
# compute_candidates needs the target lm_head attached before capture.
|
||||
# A gate-admitted quantized head is capture-safe: the target's own
|
||||
# logits path already runs the same kernel under CUDA graphs.
|
||||
if not is_dense_head_weight(
|
||||
getattr(lm_head, "weight", None)
|
||||
) and not should_apply_lm_head_quant_method(
|
||||
lm_head, getattr(lm_head, "quant_method", None)
|
||||
):
|
||||
if not head_supported:
|
||||
return _eager("unsupported quantized lm_head")
|
||||
self.draft_model.lm_head = lm_head
|
||||
if self.model_runner.tp_rank == 0:
|
||||
@@ -829,6 +845,17 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
device=self.device,
|
||||
sampling_enabled=self._selector_sampling_enabled,
|
||||
)
|
||||
if self.lilicorr is not None:
|
||||
if not head_supported:
|
||||
return _eager("unsupported quantized lm_head")
|
||||
return build_lilicorr_draft_sampler(
|
||||
sampling_enabled=self._lilicorr_sampling_enabled,
|
||||
head=self.lilicorr,
|
||||
draft_model=self.draft_model,
|
||||
embed_tokens=target_input_embeddings(target_model),
|
||||
lm_head=lm_head,
|
||||
block_size=self.block_size,
|
||||
)
|
||||
if not hasattr(lm_head, "weight"):
|
||||
return _eager("quantized lm_head has no dense weight")
|
||||
if not is_dense_head_weight(lm_head.weight):
|
||||
@@ -1736,6 +1763,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
positions: torch.Tensor,
|
||||
cache_loc_2d: Optional[torch.Tensor] = None,
|
||||
commit_lens: Optional[torch.Tensor] = None,
|
||||
extend_lens: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
"""Materialize target context features into the draft KV cache at explicit slots.
|
||||
|
||||
@@ -1835,6 +1863,13 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
),
|
||||
):
|
||||
ctx_hidden = self.draft_model.project_target_hidden(target_hidden)
|
||||
if self.lilicorr is not None:
|
||||
self._lilicorr_anchor = publish_anchor(
|
||||
draft_sampler=self._draft_sampler,
|
||||
ctx_hidden=ctx_hidden,
|
||||
extend_lens=extend_lens,
|
||||
commit_lens=commit_lens,
|
||||
)
|
||||
|
||||
if cache_loc_2d is not None:
|
||||
bs = int(commit_lens.shape[0])
|
||||
@@ -2186,6 +2221,10 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
self._warned_sampling_fallback = True
|
||||
return
|
||||
|
||||
# The sampled head hands verify the q it drew from, so no fallback is needed.
|
||||
if self.lilicorr is not None and self._lilicorr_sampling_enabled:
|
||||
return
|
||||
|
||||
if (
|
||||
not is_dflash_sampling_verify_available()
|
||||
and not self._warned_sampling_fallback
|
||||
@@ -2283,6 +2322,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
target_hidden=logits_output.hidden_states,
|
||||
cache_loc=batch.out_cache_loc,
|
||||
positions=positions,
|
||||
extend_lens=ctx_lens,
|
||||
)
|
||||
|
||||
# Avoid copying large hidden-state buffers to CPU in overlap scheduling.
|
||||
@@ -2517,7 +2557,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
global_num_token_non_padded_cpu=bs * block_size,
|
||||
)
|
||||
|
||||
if self.selector is not None:
|
||||
if self.selector is not None or self.lilicorr is not None:
|
||||
self._selector_sample = None
|
||||
if self._draft_sampler is not None:
|
||||
# Consumed by the in-graph sample; must be staged before the replay.
|
||||
@@ -2588,6 +2628,15 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
self._draft_sampler.candidate_out[:bs],
|
||||
self._draft_sampler.q_out[:bs],
|
||||
)
|
||||
elif (
|
||||
self.lilicorr is not None
|
||||
and self._lilicorr_sampling_enabled
|
||||
and not _is_all_greedy(batch.sampling_info)
|
||||
):
|
||||
self._selector_sample = (
|
||||
self._draft_sampler.candidate_out[:bs],
|
||||
self._draft_sampler.q_out[:bs],
|
||||
)
|
||||
elif self.selector is not None:
|
||||
with self.draft_tp_context(
|
||||
self.draft_model_runner.tp_group,
|
||||
@@ -2600,6 +2649,40 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
anchor_token_ids=block_ids[:, 0],
|
||||
sampling_info=batch.sampling_info,
|
||||
)
|
||||
elif self.lilicorr is not None:
|
||||
if (
|
||||
self._lilicorr_sampling_enabled
|
||||
and not self._warned_lilicorr_eager
|
||||
and self.model_runner.tp_rank == 0
|
||||
):
|
||||
logger.warning(
|
||||
"LiLiCorr sampled draft ran the eager head on a decode step "
|
||||
"(draft cuda graph unavailable for batch size %d; captured "
|
||||
"buckets do not cover it, or graphs are disabled). Acceptance is "
|
||||
"unaffected but expect a large throughput regression, and do not "
|
||||
"compare tokens/s from this run against a folded one.",
|
||||
bs,
|
||||
)
|
||||
self._warned_lilicorr_eager = True
|
||||
draft_hidden = draft_logits_output.hidden_states
|
||||
if draft_hidden is None:
|
||||
raise RuntimeError("DFLASH draft model returned no hidden states.")
|
||||
with self.draft_tp_context(self.draft_model_runner.tp_group):
|
||||
draft_next, lilicorr_candidate_ids, lilicorr_q_rows = (
|
||||
propose_lilicorr_block(
|
||||
head=self.lilicorr,
|
||||
draft_hidden=draft_hidden.view(bs, int(self.block_size), -1),
|
||||
lm_head=lm_head,
|
||||
embed_tokens=target_input_embeddings(
|
||||
self.target_worker.model_runner.model
|
||||
),
|
||||
anchor=self._lilicorr_anchor,
|
||||
sampling_info=batch.sampling_info,
|
||||
sampling_enabled=self._lilicorr_sampling_enabled,
|
||||
)
|
||||
)
|
||||
if lilicorr_q_rows is not None and not _is_all_greedy(batch.sampling_info):
|
||||
self._selector_sample = (lilicorr_candidate_ids, lilicorr_q_rows)
|
||||
else:
|
||||
draft_hidden = draft_logits_output.hidden_states
|
||||
if draft_hidden is None:
|
||||
|
||||
@@ -0,0 +1,416 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.speculative.lilicorr import MAX_FUSED_CANDIDATE_TOPK
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.models.dflash import candidate_topk
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.speculative.dflash_utils import _get_dflash_config
|
||||
from sglang.srt.speculative.dspark_components.dspark_draft import resolve_greedy_mask
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def resolve_sampling_enabled(*, device_supported: bool) -> bool:
|
||||
requested = envs.SGLANG_ENABLE_LILICORR_SAMPLING.get()
|
||||
if envs.SGLANG_LILICORR_REQUIRE_SAMPLING.get() and not requested:
|
||||
raise RuntimeError(
|
||||
"SGLANG_LILICORR_REQUIRE_SAMPLING is set but SGLANG_ENABLE_LILICORR_SAMPLING "
|
||||
"is not; this process would serve the greedy commit under a sampled arm's name."
|
||||
)
|
||||
logger.info("LiLiCorr draft commit: %s", "sampled" if requested else "greedy")
|
||||
return requested and device_supported
|
||||
|
||||
|
||||
class LiLiCorrConfig(msgspec.Struct, frozen=True):
|
||||
candidate_topk: int
|
||||
hidden_size: int
|
||||
num_layers: int
|
||||
num_heads: int
|
||||
mlp_ratio: float
|
||||
factor_dim: int
|
||||
vector_eps: float
|
||||
logit_scale: float
|
||||
|
||||
def resolve_hidden_size(self, *, model_hidden_size: int) -> int:
|
||||
# 0 means "as wide as the draft", recorded for a head with no token_proj.
|
||||
return int(self.hidden_size) if self.hidden_size else int(model_hidden_size)
|
||||
|
||||
|
||||
def _parse_lilicorr_config(dflash_cfg: dict) -> Optional[LiLiCorrConfig]:
|
||||
if not any(key.startswith("lilicorr_") for key in dflash_cfg):
|
||||
return None
|
||||
enabled = dflash_cfg.get("lilicorr_enabled")
|
||||
if enabled is not None and not bool(enabled):
|
||||
return None
|
||||
|
||||
def required(key: str, cast, *, positive: bool = True):
|
||||
# No defaults: logit_scale and vector_eps change no shape, so load cannot catch them.
|
||||
full_key = f"lilicorr_{key}"
|
||||
if full_key not in dflash_cfg:
|
||||
raise ValueError(
|
||||
f"DFLASH dflash_config.{full_key} is required to rebuild the LiLiCorr "
|
||||
"head, and the checkpoint does not carry it."
|
||||
)
|
||||
try:
|
||||
value = cast(dflash_cfg[full_key])
|
||||
except Exception as e:
|
||||
raise ValueError(
|
||||
f"Invalid dflash_config.{full_key}={dflash_cfg[full_key]!r}."
|
||||
) from e
|
||||
if positive and value <= 0:
|
||||
raise ValueError(f"dflash_config.{full_key} must be positive, got {value}.")
|
||||
return value
|
||||
|
||||
candidate_topk = required("candidate_topk", int)
|
||||
if candidate_topk & (candidate_topk - 1):
|
||||
raise ValueError(
|
||||
f"dflash_config.lilicorr_candidate_topk must be a power of two, got "
|
||||
f"{candidate_topk}: tl.arange requires a power-of-two extent."
|
||||
)
|
||||
if candidate_topk > MAX_FUSED_CANDIDATE_TOPK:
|
||||
raise ValueError(
|
||||
f"dflash_config.lilicorr_candidate_topk={candidate_topk} exceeds the fused "
|
||||
f"greedy commit's width of {MAX_FUSED_CANDIDATE_TOPK}, one Triton lane "
|
||||
"group; a wider head would serve on the reference path."
|
||||
)
|
||||
|
||||
return LiLiCorrConfig(
|
||||
candidate_topk=candidate_topk,
|
||||
hidden_size=required("hidden_size", int, positive=False),
|
||||
num_layers=required("num_layers", int),
|
||||
num_heads=required("num_heads", int),
|
||||
mlp_ratio=required("mlp_ratio", float),
|
||||
factor_dim=required("factor_dim", int),
|
||||
vector_eps=required("vector_eps", float),
|
||||
logit_scale=required("logit_scale", float),
|
||||
)
|
||||
|
||||
|
||||
def parse_lilicorr_draft_config(*, draft_hf_config: Any) -> LiLiCorrConfig:
|
||||
config = _parse_lilicorr_config(_get_dflash_config(draft_hf_config))
|
||||
if config is None:
|
||||
raise ValueError(
|
||||
"LiLiCorr requires the lilicorr_* geometry fields in dflash_config; a "
|
||||
'checkpoint declaring architectures=["LiLiCorrDraftModel"] without them '
|
||||
"cannot be rebuilt."
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def reject_added_vocab(lm_head) -> None:
|
||||
if not isinstance(lm_head, VocabParallelEmbedding):
|
||||
return
|
||||
if int(lm_head.shard_indices.num_added_elements) != 0:
|
||||
raise NotImplementedError(
|
||||
"LiLiCorr's candidate head does not support added vocabulary: those rows "
|
||||
"sit past the padded base shard, so a contiguous top-k would skip them."
|
||||
)
|
||||
|
||||
|
||||
def target_input_embeddings(target_model):
|
||||
# Not the worker's _resolve_dflash_embedding_module, which returns the draft's own
|
||||
# table for Nemotron-3.5: the head must embed with the table it was trained against.
|
||||
embed = target_model.get_input_embeddings()
|
||||
if embed is None:
|
||||
raise RuntimeError("DFLASH target model exposes no input embeddings.")
|
||||
return embed
|
||||
|
||||
|
||||
def lilicorr_candidates(
|
||||
*, hidden_states: torch.Tensor, lm_head, topk: int
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
# log_softmax(logits).topk(topk) over the FULL vocabulary: the head was trained on it.
|
||||
ids, vals, lse = candidate_topk(
|
||||
hidden_states, lm_head, int(topk), with_partition=True
|
||||
)
|
||||
return vals.float() - lse.unsqueeze(-1), ids
|
||||
|
||||
|
||||
def per_request_last_row(
|
||||
*,
|
||||
num_rows: int,
|
||||
extend_lens: Optional[torch.Tensor],
|
||||
commit_lens: Optional[torch.Tensor],
|
||||
) -> Optional[torch.Tensor]:
|
||||
# Verify's commit_lens index a [bs, block_size]-padded buffer (constant stride);
|
||||
# prefill/extend's extend_lens index a packed ragged one.
|
||||
if commit_lens is not None:
|
||||
bs = int(commit_lens.shape[0])
|
||||
if bs == 0 or num_rows % bs != 0:
|
||||
return None
|
||||
stride = num_rows // bs
|
||||
base = torch.arange(bs, device=commit_lens.device, dtype=torch.int64) * stride
|
||||
return (base + commit_lens.to(torch.int64) - 1).clamp_min(0)
|
||||
if extend_lens is None or extend_lens.numel() == 0:
|
||||
return None
|
||||
lens = extend_lens.to(torch.int64).flatten()
|
||||
return (torch.cumsum(lens, dim=0) - 1).clamp_min(0)
|
||||
|
||||
|
||||
def publish_anchor(
|
||||
*,
|
||||
draft_sampler,
|
||||
ctx_hidden: torch.Tensor,
|
||||
extend_lens: Optional[torch.Tensor] = None,
|
||||
commit_lens: Optional[torch.Tensor] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
ends = per_request_last_row(
|
||||
num_rows=int(ctx_hidden.shape[0]),
|
||||
extend_lens=extend_lens,
|
||||
commit_lens=commit_lens,
|
||||
)
|
||||
anchor = None if ends is None else ctx_hidden.index_select(0, ends)
|
||||
if draft_sampler is not None:
|
||||
draft_sampler.set_anchor(anchor, 0 if anchor is None else int(anchor.shape[0]))
|
||||
return anchor
|
||||
|
||||
|
||||
def propose_lilicorr_block(
|
||||
*,
|
||||
head,
|
||||
draft_hidden: torch.Tensor,
|
||||
lm_head,
|
||||
embed_tokens,
|
||||
anchor: Optional[torch.Tensor],
|
||||
sampling_info=None,
|
||||
sampling_enabled: bool = False,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]:
|
||||
bs, block_size, hidden_size = draft_hidden.shape
|
||||
slots = block_size - 1
|
||||
pass_hidden = draft_hidden[:, 1:, :]
|
||||
|
||||
log_probs, candidate_tokens = lilicorr_candidates(
|
||||
hidden_states=pass_hidden.reshape(bs * slots, hidden_size),
|
||||
lm_head=lm_head,
|
||||
topk=int(head.candidate_topk),
|
||||
)
|
||||
candidate_tokens = candidate_tokens.view(bs, slots, int(head.candidate_topk))
|
||||
|
||||
feat = int(head.context_proj.in_features)
|
||||
if anchor is None or int(anchor.shape[0]) != bs:
|
||||
# The batch resized since the anchor write; score invalid rather than misaligned.
|
||||
anchor_hidden = torch.zeros(
|
||||
(bs, feat), device=draft_hidden.device, dtype=draft_hidden.dtype
|
||||
)
|
||||
anchor_valid = torch.zeros((bs,), dtype=torch.bool, device=draft_hidden.device)
|
||||
else:
|
||||
anchor_hidden = anchor.view(bs, feat)
|
||||
anchor_valid = torch.ones((bs,), dtype=torch.bool, device=draft_hidden.device)
|
||||
|
||||
# Outside the head because on the target model it may be a TP-sharded collective.
|
||||
common = dict(
|
||||
token_embeddings=embed_tokens(candidate_tokens).detach(),
|
||||
candidate_tokens=candidate_tokens,
|
||||
candidate_log_probs=log_probs.view(bs, slots, int(head.candidate_topk)),
|
||||
pass_hidden=pass_hidden,
|
||||
anchor_hidden=anchor_hidden,
|
||||
anchor_valid=anchor_valid,
|
||||
)
|
||||
device = draft_hidden.device
|
||||
if sampling_enabled and sampling_info is not None:
|
||||
# A greedy row's temperature may be 0, and it divides before greedy_mask applies.
|
||||
temperatures = sampling_info.temperatures.view(-1)[:bs].float().clamp_min(1e-5)
|
||||
else:
|
||||
temperatures = torch.ones(bs, dtype=torch.float32, device=device)
|
||||
greedy_mask = (
|
||||
resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device)
|
||||
if sampling_enabled
|
||||
else torch.ones(bs, dtype=torch.bool, device=device)
|
||||
)
|
||||
selected, q_rows = head.select_with_proposal(
|
||||
uniforms=torch.rand(bs, slots, dtype=torch.float32, device=device),
|
||||
temperatures=temperatures,
|
||||
greedy_mask=greedy_mask,
|
||||
**common,
|
||||
)
|
||||
if not sampling_enabled:
|
||||
return selected.to(torch.long), None, None
|
||||
return selected.to(torch.long), candidate_tokens, q_rows
|
||||
|
||||
|
||||
class LiLiCorrDraftSampler:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
head,
|
||||
embed_tokens,
|
||||
lm_head,
|
||||
block_size: int,
|
||||
max_bs: int,
|
||||
anchor_features: int,
|
||||
sampling_enabled: bool,
|
||||
) -> None:
|
||||
self.head = head
|
||||
self.sampling_enabled = bool(sampling_enabled)
|
||||
self.embed_tokens = embed_tokens
|
||||
self.lm_head = lm_head
|
||||
self.block_size = int(block_size)
|
||||
self.slots = self.block_size - 1
|
||||
self.topk = int(head.candidate_topk)
|
||||
self.max_bs = int(max_bs)
|
||||
|
||||
# From the head, not from lm_head.weight: a quantized head's weight is packed,
|
||||
# so its dtype is not the dtype these buffers carry.
|
||||
anchor_row = next(head.parameters())
|
||||
device, dtype = anchor_row.device, anchor_row.dtype
|
||||
max_rows = self.max_bs * self.slots
|
||||
|
||||
self.out = torch.empty((max_rows,), dtype=torch.int64, device=device)
|
||||
self.anchor = torch.zeros(
|
||||
(self.max_bs, int(anchor_features)), dtype=dtype, device=device
|
||||
)
|
||||
self.anchor_valid = torch.zeros((self.max_bs,), dtype=torch.bool, device=device)
|
||||
self.token_table = head.build_token_table(embed_tokens)
|
||||
|
||||
self.temperatures = torch.ones(
|
||||
(self.max_bs,), dtype=torch.float32, device=device
|
||||
)
|
||||
self.greedy_mask = torch.ones((self.max_bs,), dtype=torch.bool, device=device)
|
||||
self.uniforms = torch.zeros(
|
||||
(self.max_bs, self.slots), dtype=torch.float32, device=device
|
||||
)
|
||||
self.q_out = torch.empty(
|
||||
(self.max_bs, self.slots, self.topk), dtype=torch.float32, device=device
|
||||
)
|
||||
self.candidate_out = torch.empty(
|
||||
(self.max_bs, self.slots, self.topk), dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
def set_anchor(self, rows: Optional[torch.Tensor], bs: int) -> None:
|
||||
# Rows past bs are zeroed, not left stale: the graph runs at the bucket size.
|
||||
# Filter/merge can hand a row a neighbour's anchor, costing acceptance only.
|
||||
count = int(bs)
|
||||
if rows is None or int(rows.shape[0]) != count or count > self.max_bs:
|
||||
count = 0
|
||||
if count:
|
||||
self.anchor[:count].copy_(rows)
|
||||
self.anchor_valid[:count].fill_(True)
|
||||
if count < self.max_bs:
|
||||
self.anchor[count:].zero_()
|
||||
self.anchor_valid[count:].fill_(False)
|
||||
|
||||
def stage_sampling_params(self, *, bs: int, sampling_info) -> None:
|
||||
if not self.sampling_enabled:
|
||||
return
|
||||
if sampling_info is None:
|
||||
self.temperatures[:bs].fill_(1.0)
|
||||
self.greedy_mask[:bs].fill_(True)
|
||||
return
|
||||
torch.clamp(
|
||||
sampling_info.temperatures.view(-1)[:bs].to(torch.float32),
|
||||
min=1e-5,
|
||||
out=self.temperatures[:bs],
|
||||
)
|
||||
self.greedy_mask[:bs].copy_(
|
||||
resolve_greedy_mask(
|
||||
bs=bs, sampling_info=sampling_info, device=self.greedy_mask.device
|
||||
)
|
||||
)
|
||||
|
||||
def __call__(self, hidden_states: torch.Tensor, input_ids=None) -> None:
|
||||
del input_ids
|
||||
bs = hidden_states.shape[0] // self.block_size
|
||||
rows = bs * self.slots
|
||||
|
||||
if bs > self.max_bs:
|
||||
raise RuntimeError(
|
||||
f"LiLiCorrDraftSampler was built for max_bs={self.max_bs} but the draft "
|
||||
f"graph replayed at bs={bs}; its static buffers are undersized."
|
||||
)
|
||||
hidden_size = hidden_states.shape[-1]
|
||||
pass_hidden = hidden_states.view(bs, self.block_size, hidden_size)[:, 1:, :]
|
||||
log_probs, candidate_tokens = lilicorr_candidates(
|
||||
hidden_states=pass_hidden.reshape(rows, hidden_size),
|
||||
lm_head=self.lm_head,
|
||||
topk=self.topk,
|
||||
)
|
||||
candidate_tokens = candidate_tokens.view(bs, self.slots, self.topk)
|
||||
|
||||
pre_projected = self.token_table is not None
|
||||
token_embeddings = (
|
||||
self.token_table[candidate_tokens]
|
||||
if pre_projected
|
||||
else self.embed_tokens(candidate_tokens).detach()
|
||||
)
|
||||
common = dict(
|
||||
token_embeddings=token_embeddings,
|
||||
candidate_tokens=candidate_tokens,
|
||||
candidate_log_probs=log_probs.view(bs, self.slots, self.topk),
|
||||
pass_hidden=pass_hidden,
|
||||
anchor_hidden=self.anchor[:bs],
|
||||
anchor_valid=self.anchor_valid[:bs],
|
||||
already_projected=pre_projected,
|
||||
)
|
||||
if self.sampling_enabled:
|
||||
self.uniforms[:bs].uniform_()
|
||||
selected, q_rows = self.head.select_with_proposal(
|
||||
uniforms=self.uniforms[:bs],
|
||||
temperatures=self.temperatures[:bs],
|
||||
greedy_mask=self.greedy_mask[:bs],
|
||||
**common,
|
||||
)
|
||||
if self.sampling_enabled:
|
||||
self.q_out[:bs].copy_(q_rows)
|
||||
self.candidate_out[:bs].copy_(candidate_tokens)
|
||||
self.out[:rows].copy_(selected.reshape(-1).to(torch.int64))
|
||||
|
||||
|
||||
def draft_graph_batch_sizes() -> list[int]:
|
||||
return sorted(
|
||||
{int(bs) for bs in get_exec().graph.cuda_graph_config.decode.bs if bs > 0}
|
||||
)
|
||||
|
||||
|
||||
def build_lilicorr_draft_sampler(
|
||||
*,
|
||||
head,
|
||||
draft_model,
|
||||
embed_tokens,
|
||||
lm_head,
|
||||
block_size: int,
|
||||
sampling_enabled: bool = False,
|
||||
) -> Optional[LiLiCorrDraftSampler]:
|
||||
def eager(reason: str) -> None:
|
||||
logger.warning(
|
||||
"LiLiCorr head kept eager (reason=%s): a bring-up path, not a serving "
|
||||
"configuration. Expect a large throughput regression and numbers that are "
|
||||
"not comparable to any published row.",
|
||||
reason,
|
||||
)
|
||||
return None
|
||||
|
||||
tp_group = get_parallel().tp_group
|
||||
if int(tp_group.world_size) != 1:
|
||||
return eager("tp>1")
|
||||
batch_sizes = draft_graph_batch_sizes()
|
||||
if not batch_sizes:
|
||||
return eager("no draft graph batch sizes to size the static buffers from")
|
||||
max_bs = batch_sizes[-1]
|
||||
|
||||
reject_added_vocab(lm_head)
|
||||
parameter = next(head.parameters())
|
||||
head.materialize_inference_buffers(parameter.device, parameter.dtype)
|
||||
|
||||
sampler = LiLiCorrDraftSampler(
|
||||
head=head,
|
||||
embed_tokens=embed_tokens,
|
||||
lm_head=lm_head,
|
||||
block_size=int(block_size),
|
||||
max_bs=int(max_bs),
|
||||
anchor_features=int(draft_model.fc.out_features),
|
||||
sampling_enabled=sampling_enabled,
|
||||
)
|
||||
logger.info(
|
||||
"LiLiCorr select folded into the draft cuda graph: max_bs=%d block_size=%d K=%d.",
|
||||
max_bs,
|
||||
int(block_size),
|
||||
sampler.topk,
|
||||
)
|
||||
return sampler
|
||||
Reference in New Issue
Block a user