From 78eee88113510d6b0d70f6978739d0845e2fff0f Mon Sep 17 00:00:00 2001 From: mrusanovsky Date: Mon, 28 Sep 2026 01:36:48 +0300 Subject: [PATCH] [Spec] Add LiLiCorr: a candidate-lattice reranker for DFlash drafts (#37462) Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: kpham-sgl Co-authored-by: Claude Opus 5.5 --- .../speculative_decoding.mdx | 34 ++ .../kernels/ops/speculative/__init__.py | 2 + .../kernels/ops/speculative/lilicorr.py | 235 +++++++++ python/sglang/srt/environ.py | 2 + python/sglang/srt/models/dflash.py | 96 ++-- python/sglang/srt/models/lilicorr.py | 473 ++++++++++++++++++ .../srt/speculative/dflash_worker_v2.py | 99 +++- .../sglang/srt/speculative/lilicorr_utils.py | 416 +++++++++++++++ .../e2e/speculative/test_dflash_lilicorr.py | 73 +++ .../ops/speculative/test_lilicorr_cuda.py | 139 +++++ .../unit/spec/test_dflash_logits.py | 29 ++ 11 files changed, 1551 insertions(+), 47 deletions(-) create mode 100644 python/sglang/kernels/ops/speculative/lilicorr.py create mode 100644 python/sglang/srt/models/lilicorr.py create mode 100644 python/sglang/srt/speculative/lilicorr_utils.py create mode 100644 test/registered/e2e/speculative/test_dflash_lilicorr.py create mode 100644 test/registered/kernels/ops/speculative/test_lilicorr_cuda.py diff --git a/docs/docs/advanced_features/speculative_decoding.mdx b/docs/docs/advanced_features/speculative_decoding.mdx index 004e9ef75b..727d091bf6 100644 --- a/docs/docs/advanced_features/speculative_decoding.mdx +++ b/docs/docs/advanced_features/speculative_decoding.mdx @@ -594,6 +594,40 @@ response = client.chat.completions.create( print(response.choices[0].message.content) ``` +### LiLiCorr: reranked DFlash drafts + +A DFlash drafter is trained on per-position marginals rather than on the joint block distribution, so its drafted tokens are individually plausible yet jointly incoherent. **LiLiCorr** ([arXiv:2608.20530](https://arxiv.org/abs/2608.20530)) keeps the top-`k` candidates per block position, scores the whole lattice jointly with a small two-layer transformer, and commits a path through it greedily instead of taking the per-slot argmax. Verify is unchanged, so outputs remain distributionally identical to the target model's and only acceptance length moves. + +LiLiCorr is **not a separate `--speculative-algorithm`**: it is a property of the draft checkpoint, the same way the DFlash candidate selector is. Serve it with `--speculative-algorithm DFLASH` and a checkpoint whose config declares `architectures: ["LiLiCorrDraftModel"]` and carries the `lilicorr_*` geometry fields inside `dflash_config`. The head is then built, loaded and folded into the draft CUDA graph automatically. + +No LiLiCorr drafter is published yet, so the draft path below is a placeholder. + +```bash Command +python3 -m sglang.launch_server \ + --model Qwen/Qwen3-8B \ + --attention-backend fa3 \ + --speculative-algorithm DFLASH \ + --speculative-draft-model-path +``` + +**Use a backend that opts out of the host `seq_lens` mirror.** `AttentionBackend.needs_cpu_seq_lens` defaults to `True`; `fa3`, `triton`, `trtllm_mha`, `trtllm_mla`, `flashmla` and `dsa` set it `False`, while `FlashInferAttnBackend` inherits the default. Under a backend that does not opt out the worker publishes `seq_lens_cpu` every step, which is a device-to-host sync per block. The flag is OR-ed over the target and draft runners, so both have to opt out. This is a property of DFlash decoding rather than of LiLiCorr, but it dominates the throughput of a 16-slot block. Backends also differ in attention numerics, so acceptance measured on different backends does not belong in one table. + +A checkpoint carrying head weights but declaring `architectures: ["DFlashDraftModel"]` loads as plain DFlash and ignores the head, which reads as a small acceptance loss rather than an error. The reverse, declaring `LiLiCorrDraftModel` without complete head weights, is refused at load. + +#### Sampling the committed path + +Above temperature 0 the target samples, but a deterministic draft is paid `p(argmax)` per slot because verify is handed a point mass. `SGLANG_ENABLE_LILICORR_SAMPLING=1` draws each slot from `softmax(scores / T)` over that slot's candidates at the request's own temperature and publishes the proposal it drew from, so rejection sampling is paid the overlap of the two distributions instead. Verify stays exact. Greedy requests sharing the batch take the argmax and report a one-hot proposal. + +Off by default; with it unset the captured body is `select` itself, so temperature-0 output is unchanged bit for bit. On Qwen3-8B it is worth about 6% acceptance and 6% end-to-end at temperature 1, at under 1% of the per-block wall. + +**Only enable it on the folded path.** The eager fallback that serves the steps the draft CUDA graph cannot is correct but costs a large fraction of throughput, and because acceptance is identical either way that regression shows up only in tokens/s. `build_lilicorr_draft_sampler` warns with a reason when it declines to fold, and the worker warns once if a decode step lands on the eager head. + +#### Drafters with grouped convolutions + +LiLiCorr composes with the grouped dynamic convolutions that wrap each DFlash sublayer, and serving one needs no extra flag: the convolution belongs to the DFlash backbone, so `LiLiCorrDraftModel` inherits the builder and picks it up from `conv_kernel_size` and `conv_group_size` in `dflash_config`. + +Both keys default to `0`, so a checkpoint whose config lost them builds no convolution modules and the backbone loader then drops every convolution tensor it cannot resolve. That is correct for rotary caches and the worst case here, because the draft would serve as its convolution-free parent at a lower but entirely believable acceptance length. The loader therefore refuses any checkpoint whose convolution tensors and built modules disagree, in either direction. + --- ## Standalone Speculative Decoding (Small Draft Model) diff --git a/python/sglang/kernels/ops/speculative/__init__.py b/python/sglang/kernels/ops/speculative/__init__.py index fa44cf5c9f..b4d51f651c 100644 --- a/python/sglang/kernels/ops/speculative/__init__.py +++ b/python/sglang/kernels/ops/speculative/__init__.py @@ -21,6 +21,8 @@ _TRITON_KERNELS = [ ("topk1", "draft_topk1_postprocess"), ("ragged_verify_kernels", "pad_verify_lens_to_bucket"), ("ragged_verify_kernels", "build_qo_indptr"), + ("lilicorr", "lilicorr_topk_lse"), + ("lilicorr", "lilicorr_sample_path"), ("reject_sampling", "chain_speculative_sampling_triton"), ("row_argmax", "row_argmax"), ("reconstruct_tree", "reconstruct_indices_from_tree_mask_triton"), diff --git a/python/sglang/kernels/ops/speculative/lilicorr.py b/python/sglang/kernels/ops/speculative/lilicorr.py new file mode 100644 index 0000000000..861b6c383b --- /dev/null +++ b/python/sglang/kernels/ops/speculative/lilicorr.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +from typing import Tuple + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl + +from sglang.kernels.ops.speculative.dflash import selector_walk_triton + +# Launch geometry tuned on H100 for a ~150k vocabulary; TILE * TPP is the load block. +_TILE = 1024 +_TILES_PER_PROGRAM = 8 +_NUM_WARPS = 4 + +# One Triton lane group in the selector walk; the config refuses a wider head. +MAX_FUSED_CANDIDATE_TOPK = 16 + +_NEG = -3.0e38 + + +@triton.jit +def _tiled_scan( + lp, # [N, V] logits (any float dtype) + tmax, # [N, T] fp32 per-tile max + pm, # [N, P] fp32 online-softmax running max + ps, # [N, P] fp32 online-softmax running sum + V, + T, + stride_lp_n, + stride_tmax_n, + stride_p_n, + TILE: tl.constexpr, + TPP: tl.constexpr, +): + neg = -3.0e38 + row = tl.program_id(0) + grp = tl.program_id(1) + + t0 = grp * TPP + tile_off = t0 + tl.arange(0, TPP) + offs = tile_off[:, None] * TILE + tl.arange(0, TILE)[None, :] + mask = (tile_off[:, None] < T) & (offs < V) + + x = tl.load(lp + row * stride_lp_n + offs, mask=mask, other=neg) + x = x.to(tl.float32) + + tm = tl.max(x, axis=1) + tl.store(tmax + row * stride_tmax_n + tile_off, tm, mask=tile_off < T) + + m = tl.max(tm, axis=0) + s = tl.sum(tl.exp(x - m), axis=0) + s = tl.sum(s, axis=0) + tl.store(pm + row * stride_p_n + grp, m) + tl.store(ps + row * stride_p_n + grp, s) + + +@triton.jit +def _tiled_select( + lp, # [N, V] logits + tids, # [N, KT] int64 candidate tile ids (ascending) + ov, # [N, K] fp32 out vals (descending) + oi, # [N, K] int64 out ids + V, + stride_lp_n, + stride_tid_n, + stride_ov_n, + stride_oi_n, + TILE: tl.constexpr, + KT: tl.constexpr, + K: tl.constexpr, + NSEL: tl.constexpr, +): + neg = -3.0e38 + row = tl.program_id(0) + + tid = tl.load(tids + row * stride_tid_n + tl.arange(0, KT)) + offs2 = tid[:, None] * TILE + tl.arange(0, TILE)[None, :] + mask2 = offs2 < V + x2 = tl.load(lp + row * stride_lp_n + offs2, mask=mask2, other=neg) + + x = tl.reshape(x2.to(tl.float32), (NSEL,)) + ids = tl.reshape(offs2, (NSEL,)) + lane = tl.arange(0, NSEL) + + for j in range(K): + mv = tl.max(x, axis=0) + mp = tl.argmax(x, axis=0) + mid = tl.sum(tl.where(lane == mp, ids, 0), axis=0) + tl.store(ov + row * stride_ov_n + j, mv) + tl.store(oi + row * stride_oi_n + j, mid) + x = tl.where(lane == mp, neg, x) + + +def _combine_lse(pm: torch.Tensor, ps: torch.Tensor) -> torch.Tensor: + gm = pm.max(dim=-1, keepdim=True).values + s = (ps * (pm - gm).exp()).sum(dim=-1) + return gm.squeeze(-1) + s.log() + + +def _topk_lse_torch( + logits: torch.Tensor, k: int +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + vals, ids = torch.topk(logits, k, dim=-1) + rowmax = vals[:, 0:1] # topk is sorted descending + sumexp = (logits - rowmax).exp().sum(dim=-1, dtype=torch.float32) + lse = rowmax.squeeze(-1).to(torch.float32) + sumexp.log() + return vals.float(), ids.to(torch.int64), lse + + +def lilicorr_topk_lse( + logits: torch.Tensor, k: int +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + # Tile pre-selection is exact: a top-k element's tile has a max at least as large, + # so it is among the k largest-max tiles. + if not logits.is_cuda: + return _topk_lse_torch(logits, k) + + logits = logits.contiguous() + n, V = int(logits.shape[0]), int(logits.shape[1]) + K = int(k) + T = (V + _TILE - 1) // _TILE + if min(K, T) & (min(K, T) - 1): + return _topk_lse_torch(logits, k) + P = (T + _TILES_PER_PROGRAM - 1) // _TILES_PER_PROGRAM + device = logits.device + + tmax = torch.full((n, T), _NEG, dtype=torch.float32, device=device) + pm = torch.full((n, P), _NEG, dtype=torch.float32, device=device) + ps = torch.zeros((n, P), dtype=torch.float32, device=device) + _tiled_scan[(n, P)]( + logits, + tmax, + pm, + ps, + V, + T, + logits.stride(0), + tmax.stride(0), + pm.stride(0), + TILE=_TILE, + TPP=_TILES_PER_PROGRAM, + num_warps=_NUM_WARPS, + ) + + kt = min(K, T) + tids = torch.topk(tmax, kt, dim=-1).indices + tids = torch.sort(tids, dim=-1).values.to(torch.int64) + + ov = torch.empty((n, K), dtype=torch.float32, device=device) + oi = torch.empty((n, K), dtype=torch.int64, device=device) + _tiled_select[(n,)]( + logits, + tids, + ov, + oi, + V, + logits.stride(0), + tids.stride(0), + ov.stride(0), + oi.stride(0), + TILE=_TILE, + KT=kt, + K=K, + NSEL=kt * _TILE, + num_warps=_NUM_WARPS, + ) + return ov, oi, _combine_lse(pm, ps) + + +def _lattice_scores(log_start: torch.Tensor, log_pair: torch.Tensor) -> torch.Tensor: + # The walk reads only scores[:, 0, 0, :] at slot 0, so broadcasting is sound. + topk = int(log_start.shape[-1]) + start = log_start.float()[:, None, None, :].expand(-1, 1, topk, topk) + return torch.cat([start, log_pair.float()], dim=1) + + +def _selector_walk_torch( + *, + candidate_ids: torch.Tensor, + scores: torch.Tensor, + uniforms: torch.Tensor, + temperatures: torch.Tensor, + greedy_mask: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + _, num_slots, topk = candidate_ids.shape + temps = temperatures.view(-1, 1).to(torch.float32) + greedy = greedy_mask.view(-1) + previous = torch.zeros(greedy.shape[0], dtype=torch.int64, device=scores.device) + tokens, q_rows = [], [] + for slot in range(num_slots): + node = torch.gather( + scores[:, slot].float(), 1, previous.view(-1, 1, 1).expand(-1, 1, topk) + ).squeeze(1) + probs = torch.softmax(node / temps, dim=-1) + sampled = ( + uniforms[:, slot : slot + 1] + .ge(probs.cumsum(dim=-1)) + .sum(dim=-1) + .clamp_max(topk - 1) + ) + previous = torch.where(greedy, node.argmax(dim=-1), sampled) + q_rows.append( + torch.where( + greedy.unsqueeze(-1), + F.one_hot(previous, topk).to(torch.float32), + probs, + ) + ) + tokens.append( + torch.gather(candidate_ids[:, slot], 1, previous.view(-1, 1)).squeeze(1) + ) + return torch.stack(tokens, dim=-1).to(torch.int64), torch.stack(q_rows, dim=1) + + +def lilicorr_sample_path( + log_start: torch.Tensor, + log_pair: torch.Tensor, + candidate_tokens: torch.Tensor, + uniforms: torch.Tensor, + temperatures: torch.Tensor, + greedy_mask: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + # No log-prob prior in the proposal: the head was trained without one. Greedy rows + # report a point mass so min(1, p/q) stays the right acceptance test. + scores = _lattice_scores(log_start, log_pair) + walk = selector_walk_triton if scores.is_cuda else _selector_walk_torch + return walk( + candidate_ids=candidate_tokens, + scores=scores, + uniforms=uniforms, + temperatures=temperatures, + greedy_mask=greedy_mask, + ) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 11ef39e53c..9af31cae81 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1366,6 +1366,8 @@ class Envs: SGLANG_OPT_FUSED_KDA_VERIFY = EnvBool(False) # A/B: keep the DFLASH draft greedy head eager (not folded in-graph). SGLANG_DFLASH_EAGER_DRAFT_SAMPLER = EnvBool(False) + SGLANG_ENABLE_LILICORR_SAMPLING = EnvBool(False) + SGLANG_LILICORR_REQUIRE_SAMPLING = EnvBool(False) SGLANG_RAGGED_VERIFY_MODE = EnvStr("static") SGLANG_TEST_RAGGED_VERIFY_FORCE_UNIFORM_CAPTURE = EnvBool(False) # Skip draft_extend while adaptive spec is at steps=0 (drafting disabled). diff --git a/python/sglang/srt/models/dflash.py b/python/sglang/srt/models/dflash.py index 5722b43d45..f3205f3de4 100644 --- a/python/sglang/srt/models/dflash.py +++ b/python/sglang/srt/models/dflash.py @@ -12,6 +12,7 @@ import torch.nn.functional as F from torch import nn from sglang.kernels.ops.speculative.dflash import selector_walk_triton +from sglang.kernels.ops.speculative.lilicorr import lilicorr_topk_lse from sglang.srt.configs.laguna import normalize_gating from sglang.srt.distributed.communication_op import tensor_model_parallel_all_gather from sglang.srt.layers.activation import SiluAndMul @@ -101,6 +102,56 @@ def _project_candidate_logits( return logits +def candidate_topk( + hidden: torch.Tensor, + lm_head: nn.Module, + k: int, + *, + with_partition: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """Global top-k candidates from the target head: hidden [N, H] -> ids / vals [N, K]. + + Under TP (vocab-sharded lm_head): local top-k per shard, all-gather K logits/ids (not + the full vocab), then a global top-k -- identical candidates at O(tp*K) instead of + O(vocab) gather bandwidth. with_partition also returns the full-vocabulary lse [N] fp32. + """ + # The worker screens the head before capture, but the eager fallbacks attach + # whatever the target has. + weight = getattr(lm_head, "weight", None) + quant_method = getattr(lm_head, "quant_method", None) + use_quant_head = should_apply_lm_head_quant_method(lm_head, quant_method) + if not use_quant_head and not is_dense_head_weight(weight): + raise RuntimeError( + "DFlash candidate top-k requires a dense FP16/BF16/FP32 target lm_head " + "or a supported lm_head.quant_method." + ) + tp = get_parallel().tp_size == 1 + num_org = ( + int(lm_head.org_vocab_size) + if tp + else int(lm_head.shard_indices.num_org_elements) + ) + logits = _project_candidate_logits( + hidden, lm_head, num_org=num_org, use_quant_head=use_quant_head + ) + lse = None + if with_partition: + vals, ids, lse = lilicorr_topk_lse(logits, k) + else: + vals, ids = _radix_topk(logits, k) + if tp: + return ids.long(), vals, lse + global_ids = ids.long() + int(lm_head.shard_indices.org_vocab_start_index) + gathered_vals = tensor_model_parallel_all_gather(vals.float(), dim=-1) + gathered_ids = tensor_model_parallel_all_gather(global_ids, dim=-1) + top_vals, sel = torch.topk(gathered_vals, k, dim=-1) + if lse is not None: + lse = torch.logsumexp( + tensor_model_parallel_all_gather(lse.unsqueeze(-1), dim=-1), dim=-1 + ) + return torch.gather(gathered_ids, -1, sel).long(), top_vals, lse + + def _get_dflash_attention_type(config, *, default: AttentionType) -> AttentionType: """Honor explicit causality while preserving legacy layer defaults.""" text_config = config.get_text_config() @@ -579,11 +630,13 @@ class DFlashDraftModel(nn.Module): hidden_size = int(config.hidden_size) num_layers = int(config.num_hidden_layers) rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-6)) + self.rms_norm_eps = rms_norm_eps draft_config = self.draft_config = parse_dflash_draft_config( draft_hf_config=config ) self.block_size = draft_config.resolve_block_size(default=16) self.candidate_selector: Optional[nn.Module] = None + self.lilicorr: Optional[nn.Module] = None self.is_nemotron_35_draft = is_nemotron_35_draft_config(config) self.embed_tokens: Optional[VocabParallelEmbedding] = None if self.is_nemotron_35_draft: @@ -1176,47 +1229,12 @@ class DFlash2DraftModel(DFlashDraftModel): self, hidden: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: """Top-k base candidates via the target lm_head: hidden [N, H] -> global - candidate_ids / unary_logits [N, K]. Under TP (vocab-sharded lm_head): local top-k - per shard, all-gather K logits/ids (not the full vocab), then a global top-k -- - identical candidates at O(tp*K) instead of O(vocab) gather bandwidth.""" + candidate_ids / unary_logits [N, K].""" assert self.lm_head is not None, "draft_model.lm_head unset before capture" - k = self.candidate_selector.top_k - # The worker screens the head before capture, but its eager fallback - # (_propose_selector_block) attaches whatever the target has. - weight = getattr(self.lm_head, "weight", None) - quant_method = getattr(self.lm_head, "quant_method", None) - use_quant_head = should_apply_lm_head_quant_method(self.lm_head, quant_method) - if not use_quant_head and not is_dense_head_weight(weight): - raise RuntimeError( - "DFlash2 selector requires a dense FP16/BF16/FP32 target lm_head " - "or a supported lm_head.quant_method." - ) - if get_parallel().tp_size == 1: - org = int(self.lm_head.org_vocab_size) - vals, ids = _radix_topk( - _project_candidate_logits( - hidden, self.lm_head, num_org=org, use_quant_head=use_quant_head - ), - k, - ) - return ids.long(), self._transform_unary_logits(vals) - shard = self.lm_head.shard_indices - vals, ids = _radix_topk( - _project_candidate_logits( - hidden, - self.lm_head, - num_org=int(shard.num_org_elements), - use_quant_head=use_quant_head, - ), - k, - ) - global_ids = ids.long() + int(shard.org_vocab_start_index) - gathered_vals = tensor_model_parallel_all_gather(vals.float(), dim=-1) - gathered_ids = tensor_model_parallel_all_gather(global_ids, dim=-1) - top_vals, sel = torch.topk(gathered_vals, k, dim=-1) - return torch.gather(gathered_ids, -1, sel).long(), self._transform_unary_logits( - top_vals + ids, vals, _ = candidate_topk( + hidden, self.lm_head, self.candidate_selector.top_k ) + return ids, self._transform_unary_logits(vals) class MuseGlimmerAssistantModel(DFlashDraftModel): diff --git a/python/sglang/srt/models/lilicorr.py b/python/sglang/srt/models/lilicorr.py new file mode 100644 index 0000000000..27134f1dd0 --- /dev/null +++ b/python/sglang/srt/models/lilicorr.py @@ -0,0 +1,473 @@ +from __future__ import annotations + +import logging +from typing import Iterable, Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn + +from sglang.kernels.ops.speculative.lilicorr import lilicorr_sample_path +from sglang.srt.layers.layernorm import RMSNorm +from sglang.srt.models.dflash import DFlashDraftModel +from sglang.srt.speculative.lilicorr_utils import ( + LiLiCorrConfig, + parse_lilicorr_draft_config, +) + +logger = logging.getLogger(__name__) + + +class LiLiCorrLatticeAttention(nn.Module): + # nn.MultiheadAttention's parameter layout, run via SDPA since that module is not capturable. + + def __init__(self, hidden_size: int, num_heads: int) -> None: + super().__init__() + if hidden_size % num_heads != 0: + raise ValueError( + f"LiLiCorr hidden_size={hidden_size} must be divisible by " + f"num_heads={num_heads}." + ) + self.hidden_size = int(hidden_size) + self.num_heads = int(num_heads) + self.head_dim = self.hidden_size // self.num_heads + self.in_proj_weight = nn.Parameter(torch.zeros(3 * hidden_size, hidden_size)) + self.in_proj_bias = nn.Parameter(torch.zeros(3 * hidden_size)) + self.out_proj = nn.Linear(hidden_size, hidden_size) + + def forward( + self, hidden_states: torch.Tensor, attention_bias: torch.Tensor + ) -> torch.Tensor: + bsz, seq_len, _ = hidden_states.shape + qkv = F.linear(hidden_states, self.in_proj_weight, self.in_proj_bias) + q, k, v = qkv.chunk(3, dim=-1) + shape = (bsz, seq_len, self.num_heads, self.head_dim) + q = q.view(shape).transpose(1, 2) + k = k.view(shape).transpose(1, 2) + v = v.view(shape).transpose(1, 2) + # attention_bias is [1, heads, L, L]; SDPA broadcasts it over the batch. + out = F.scaled_dot_product_attention(q, k, v, attn_mask=attention_bias) + return self.out_proj( + out.transpose(1, 2).reshape(bsz, seq_len, self.hidden_size) + ) + + +class LiLiCorrLayer(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float, + rms_norm_eps: float, + ) -> None: + super().__init__() + self.attn_norm = RMSNorm(hidden_size, eps=rms_norm_eps) + self.attn = LiLiCorrLatticeAttention(hidden_size, num_heads) + self.mlp_norm = RMSNorm(hidden_size, eps=rms_norm_eps) + mlp_hidden_size = int(hidden_size * mlp_ratio) + self.mlp = nn.Sequential( + nn.Linear(hidden_size, mlp_hidden_size), + nn.SiLU(), + nn.Linear(mlp_hidden_size, hidden_size), + ) + + def forward( + self, hidden_states: torch.Tensor, attention_bias: torch.Tensor + ) -> torch.Tensor: + hidden_states = hidden_states + self.attn( + self.attn_norm(hidden_states), attention_bias + ) + return hidden_states + self.mlp(self.mlp_norm(hidden_states)) + + +class LiLiCorrHead(nn.Module): + # [log_probs, probs, logprob_gap, rank_frac, is_top1], in the trained Linear's order. + num_candidate_features = 5 + + def __init__( + self, + *, + model_hidden_size: int, + block_size: int, + rms_norm_eps: float, + config: LiLiCorrConfig, + ) -> None: + super().__init__() + hidden_size = config.resolve_hidden_size(model_hidden_size=model_hidden_size) + self.block_size = int(block_size) + self.num_candidate_slots = self.block_size - 1 + self.num_active_slots = self.num_candidate_slots + self.candidate_topk = int(config.candidate_topk) + self.hidden_size = hidden_size + self.num_heads = int(config.num_heads) + self.mlp_ratio = float(config.mlp_ratio) + self.factor_dim = int(config.factor_dim) + self.vector_eps = float(config.vector_eps) + self.logit_scale = float(config.logit_scale) + + self.token_proj = ( + nn.Identity() + if model_hidden_size == hidden_size + else nn.Linear(model_hidden_size, hidden_size) + ) + self.pass_hidden_proj = nn.Linear(model_hidden_size, hidden_size) + self.feature_mlp = nn.Sequential( + nn.LayerNorm(self.num_candidate_features), + nn.Linear(self.num_candidate_features, hidden_size), + nn.SiLU(), + nn.Linear(hidden_size, hidden_size), + ) + self.slot_embedding = nn.Parameter( + torch.zeros(1, 1, self.num_candidate_slots, 1, hidden_size) + ) + self.rank_embedding = nn.Parameter( + torch.zeros(1, 1, 1, self.candidate_topk, hidden_size) + ) + self.relative_slot_bias = nn.Parameter( + torch.zeros(self.num_heads, 2 * self.block_size - 1) + ) + self.same_slot_bias = nn.Parameter(torch.zeros(self.num_heads)) + self.context_proj = nn.Linear(model_hidden_size, hidden_size) + self.layers = nn.ModuleList( + [ + LiLiCorrLayer( + hidden_size=hidden_size, + num_heads=self.num_heads, + mlp_ratio=self.mlp_ratio, + rms_norm_eps=rms_norm_eps, + ) + for _ in range(int(config.num_layers)) + ] + ) + self.output_norm = RMSNorm(hidden_size, eps=rms_norm_eps) + self.anchor_norm = RMSNorm(hidden_size, eps=rms_norm_eps) + self.factor_input_proj = nn.Linear(hidden_size * 3, hidden_size) + self.out_head = nn.Linear(hidden_size, self.factor_dim) + self.in_head = nn.Linear(hidden_size, self.factor_dim) + self.anchor_out_head = nn.Linear(hidden_size, self.factor_dim) + + self._attn_bias: Optional[torch.Tensor] = None + self._fused_edge_weight: Optional[torch.Tensor] = None + self._fused_edge_bias: Optional[torch.Tensor] = None + self._factor_input_splits: Optional[Tuple[torch.Tensor, ...]] = None + self._rank_frac_col: Optional[torch.Tensor] = None + self._is_top1_col: Optional[torch.Tensor] = None + + @torch.no_grad() + def materialize_inference_buffers( + self, device: torch.device, dtype: torch.dtype + ) -> None: + # After weight load and before graph capture: host work, not capturable. + topk = self.candidate_topk + self._attn_bias = self._build_attention_bias(device=device, dtype=dtype) + + self._fused_edge_weight = ( + torch.cat([self.out_head.weight, self.in_head.weight], dim=0) + .to(device=device, dtype=dtype) + .contiguous() + ) + self._fused_edge_bias = ( + torch.cat([self.out_head.bias, self.in_head.bias], dim=0) + .to(device=device, dtype=dtype) + .contiguous() + ) + + weight = self.factor_input_proj.weight + hdim = self.hidden_size + self._factor_input_splits = ( + weight[:, :hdim].contiguous(), + weight[:, hdim : 2 * hdim].contiguous(), + weight[:, 2 * hdim :].contiguous(), + ) + + if topk > 1: + rank_frac = torch.arange(topk, device=device, dtype=torch.float32).view( + 1, 1, 1, topk + ) / float(topk - 1) + else: + rank_frac = torch.zeros(1, 1, 1, topk, device=device, dtype=torch.float32) + is_top1 = torch.zeros(1, 1, 1, topk, device=device, dtype=torch.float32) + is_top1[..., 0] = 1.0 + self._rank_frac_col = rank_frac.contiguous() + self._is_top1_col = is_top1.contiguous() + + def _build_attention_bias( + self, *, device: torch.device, dtype: torch.dtype + ) -> torch.Tensor: + topk = self.candidate_topk + slot_ids = torch.arange( + self.num_active_slots, device=device, dtype=torch.long + ).repeat_interleave(topk) + rel = slot_ids.view(-1, 1) - slot_ids.view(1, -1) + rel = rel.clamp(min=-(self.block_size - 1), max=self.block_size - 1) + bias = self.relative_slot_bias[:, rel + self.block_size - 1] + same_slot = slot_ids.view(-1, 1) == slot_ids.view(1, -1) + bias = bias + same_slot.unsqueeze(0).to(dtype=bias.dtype) * ( + self.same_slot_bias.view(-1, 1, 1) + ) + return bias.to(device=device, dtype=dtype).contiguous() + + def set_active_block_size(self, block_size: int) -> None: + # A prefix of the trained slots; every relative offset it needs was trained. + if not 2 <= int(block_size) <= self.block_size: + raise ValueError( + f"block_size={int(block_size)} is outside [2, {self.block_size}], " + "the trained LiLiCorr block." + ) + self.num_active_slots = int(block_size) - 1 + if self._attn_bias is not None: + self._attn_bias = self._build_attention_bias( + device=self._attn_bias.device, dtype=self._attn_bias.dtype + ) + + def _require_materialized(self) -> None: + if self._attn_bias is None: + raise RuntimeError( + "LiLiCorr head scored before materialize_inference_buffers(): its " + "attention bias and fused edge heads are unbuilt." + ) + + def _project_anchor( + self, anchor_hidden: torch.Tensor, anchor_valid: torch.Tensor + ) -> torch.Tensor: + # Branch-free so there is no host sync inside the captured region. + anchor = self.context_proj(anchor_hidden) + return anchor * anchor_valid.unsqueeze(-1).to(anchor.dtype) + + def score( + self, + *, + token_embeddings: torch.Tensor, + candidate_log_probs: torch.Tensor, + pass_hidden: torch.Tensor, + anchor_hidden: torch.Tensor, + anchor_valid: torch.Tensor, + already_projected: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor]: + self._require_materialized() + bsz, n_blocks, n_slots, topk = candidate_log_probs.shape + if topk != self.candidate_topk: + raise ValueError( + f"LiLiCorr was built for candidate_topk={self.candidate_topk} but the " + f"lattice carries {topk}; both are sized for the trained width." + ) + + proj_dtype = self.pass_hidden_proj.weight.dtype + if token_embeddings.dtype != proj_dtype: + token_embeddings = token_embeddings.to(proj_dtype) + if pass_hidden.dtype != proj_dtype: + pass_hidden = pass_hidden.to(proj_dtype) + + token_states = ( + token_embeddings if already_projected else self.token_proj(token_embeddings) + ) + pass_states = self.pass_hidden_proj(pass_hidden).unsqueeze(-2) + + log_probs = candidate_log_probs.float() + features = torch.stack( + [ + log_probs, + log_probs.exp(), + log_probs - log_probs.max(dim=-1, keepdim=True).values, + self._rank_frac_col.expand_as(log_probs), + self._is_top1_col.expand_as(log_probs), + ], + dim=-1, + ) + hidden_states = token_states + pass_states + hidden_states = hidden_states + self.feature_mlp( + features.to(dtype=token_states.dtype) + ) + hidden_states = hidden_states + self.slot_embedding[:, :, :n_slots] + hidden_states = hidden_states + self.rank_embedding + hidden_states = hidden_states.reshape( + bsz * n_blocks, n_slots * topk, self.hidden_size + ) + + anchor_state = self._project_anchor(anchor_hidden, anchor_valid) + attention_bias = self._attn_bias.unsqueeze(0) + for layer in self.layers: + hidden_states = layer(hidden_states, attention_bias) + hidden_states = self.output_norm(hidden_states).reshape( + bsz, n_blocks, n_slots, topk, self.hidden_size + ) + anchor_state = self.anchor_norm(anchor_state) + + w_self, w_anchor, w_cross = self._factor_input_splits + anchor_row = anchor_state[:, :, None, None, :] + pre = F.linear(hidden_states, w_self, self.factor_input_proj.bias) + pre = pre + F.linear(anchor_row, w_anchor) + pre = pre + F.linear(hidden_states * anchor_row, w_cross) + factor_hidden = F.silu(pre) + + edges = F.linear(factor_hidden, self._fused_edge_weight, self._fused_edge_bias) + out_vec, in_vec = F.normalize( + edges.unflatten(-1, (2, self.factor_dim)), + dim=-1, + eps=self.vector_eps, + ).unbind(-2) + anchor_out = F.normalize( + self.anchor_out_head(anchor_state), dim=-1, eps=self.vector_eps + ) + + start_scores = (anchor_out[:, :, None, :] * in_vec[:, :, 0, :, :]).sum(dim=-1) + pair_scores = torch.matmul( + out_vec[:, :, :-1], in_vec[:, :, 1:].transpose(-1, -2) + ) + return start_scores, pair_scores + + def log_factors( + self, start_scores: torch.Tensor, pair_scores: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + return ( + self.logit_scale * start_scores.float(), + self.logit_scale * pair_scores.float(), + ) + + def select_with_proposal( + self, + *, + token_embeddings: torch.Tensor, + candidate_tokens: torch.Tensor, + candidate_log_probs: torch.Tensor, + pass_hidden: torch.Tensor, + anchor_hidden: torch.Tensor, + anchor_valid: torch.Tensor, + uniforms: torch.Tensor, + temperatures: torch.Tensor, + greedy_mask: torch.Tensor, + already_projected: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor]: + start_scores, pair_scores = self.score( + token_embeddings=token_embeddings.unsqueeze(1), + candidate_log_probs=candidate_log_probs.unsqueeze(1), + pass_hidden=pass_hidden.unsqueeze(1), + anchor_hidden=anchor_hidden.unsqueeze(1), + anchor_valid=anchor_valid.unsqueeze(1), + already_projected=already_projected, + ) + log_start, log_pair = self.log_factors(start_scores, pair_scores) + return lilicorr_sample_path( + log_start[:, 0, :], + log_pair[:, 0], + candidate_tokens, + uniforms=uniforms, + temperatures=temperatures, + greedy_mask=greedy_mask, + ) + + @torch.no_grad() + def build_token_table(self, embed_tokens: nn.Module) -> Optional[torch.Tensor]: + if isinstance(self.token_proj, nn.Identity): + return None + weight = embed_tokens.weight + if int(self.token_proj.weight.shape[1]) != int(weight.shape[1]): + return None + return F.linear( + weight.to(self.token_proj.weight.dtype), + self.token_proj.weight, + self.token_proj.bias, + ).contiguous() + + +def check_head_weight_coverage(head: LiLiCorrHead, seen: set) -> None: + # The base loader silently drops unresolved weights, which would only show + # up as a lower acceptance length. + expected = {f"lilicorr.{name}" for name, _ in head.named_parameters()} + missing = sorted(expected - seen) + if missing: + raise ValueError( + f"LiLiCorr checkpoint is missing {len(missing)} head parameters " + f"(e.g. {missing[:5]}); a draft with no LiLiCorr head should declare " + 'architectures=["DFlashDraftModel"].' + ) + unexpected = sorted(seen - expected) + if unexpected: + raise ValueError( + f"LiLiCorr checkpoint carries {len(unexpected)} head tensors this head " + f"has no parameter for (e.g. {unexpected[:5]}). Check " + "lilicorr_hidden_size, lilicorr_num_layers and lilicorr_factor_dim in " + "dflash_config against the checkpoint." + ) + + +def check_conv_weight_coverage(model: DFlashDraftModel, seen: set) -> None: + # conv_kernel_size / conv_group_size default to 0, so a config that lost them + # builds no conv modules and the loader drops every conv tensor silently. + expected = { + name + for name, _ in model.named_parameters() + if ".attention_conv." in name or ".mlp_conv." in name + } + + if seen and not expected: + raise ValueError( + f"Draft checkpoint carries {len(seen)} grouped-convolution tensors " + f"(e.g. {sorted(seen)[:3]}) but this draft built no convolution modules: " + "dflash_config is missing conv_kernel_size / conv_group_size, which both " + "default to 0 and cannot be inferred from the tensors." + ) + if expected and not seen: + raise ValueError( + f"This draft built {len(expected)} grouped-convolution parameters from " + "dflash_config, but the checkpoint carries none, so kernel_projection " + "would serve at its random initialization." + ) + + missing = sorted(expected - seen) + unexpected = sorted(seen - expected) + if missing or unexpected: + raise ValueError( + "Draft checkpoint's grouped-convolution tensors do not correspond to the " + f"built ones: {len(missing)} missing (e.g. {missing[:3]}), " + f"{len(unexpected)} unexpected (e.g. {unexpected[:3]}). Check " + "conv_kernel_size, conv_group_size and num_hidden_layers in dflash_config." + ) + + if expected: + conv = model.layers[0].attention_conv + logger.info( + "DFLASH grouped convolution live: %d taps, group size %d, %d tensors.", + int(conv.taps), + int(conv.group_size), + len(expected), + ) + + +class LiLiCorrDraftModel(DFlashDraftModel): + def __init__(self, config, quant_config=None, prefix: str = "") -> None: + super().__init__(config=config, quant_config=quant_config, prefix=prefix) + self.lilicorr = LiLiCorrHead( + model_hidden_size=int(config.hidden_size), + block_size=int(self.block_size), + rms_norm_eps=self.rms_norm_eps, + config=parse_lilicorr_draft_config(draft_hf_config=config), + ) + + def set_block_size(self, block_size: int) -> None: + super().set_block_size(block_size) + self.lilicorr.set_active_block_size(block_size) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + seen: set[str] = set() + seen_conv: set[str] = set() + + def tracking(): + for name, weight in weights: + stripped = name[len("model.") :] if name.startswith("model.") else name + if stripped.startswith("lilicorr."): + seen.add(stripped) + elif ".attention_conv." in stripped or ".mlp_conv." in stripped: + seen_conv.add(stripped) + yield name, weight + + super().load_weights(tracking()) + + check_head_weight_coverage(self.lilicorr, seen) + check_conv_weight_coverage(self, seen_conv) + + parameter = next(self.lilicorr.parameters()) + self.lilicorr.materialize_inference_buffers(parameter.device, parameter.dtype) + + +EntryClass = [LiLiCorrDraftModel] diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index b16f024d1b..35361a58a5 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -74,6 +74,13 @@ from sglang.srt.speculative.draft_worker_common import ( make_draft_sampler_capture_hook, ) from sglang.srt.speculative.dspark_components.dspark_draft import resolve_greedy_mask +from sglang.srt.speculative.lilicorr_utils import ( + build_lilicorr_draft_sampler, + propose_lilicorr_block, + publish_anchor, + resolve_sampling_enabled, + target_input_embeddings, +) from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite from sglang.srt.speculative.spec_utils import ( @@ -386,6 +393,7 @@ class DFlashWorkerV2(BaseSpecWorker): self.device = target_worker.device self._warned_sampling_fallback = False + self._warned_lilicorr_eager = False self._draft_probs_buf = None self._logged_first_verify = False self._full_embed_gpu: Optional[torch.Tensor] = None @@ -425,6 +433,12 @@ class DFlashWorkerV2(BaseSpecWorker): self.selector = self.draft_model.candidate_selector # Ascend keeps selector proposal aligned with its greedy-only verify path. self._selector_sampling_enabled = not _is_npu + self.lilicorr = self.draft_model.lilicorr + # The sampled commit rides the selector's accept path. + self._lilicorr_sampling_enabled = self.lilicorr is not None and ( + resolve_sampling_enabled(device_supported=self._selector_sampling_enabled) + ) + self._lilicorr_anchor: Optional[torch.Tensor] = None draft_config = parse_dflash_draft_config( draft_hf_config=self.draft_model_runner.model_config.hf_config ) @@ -805,15 +819,17 @@ class DFlashWorkerV2(BaseSpecWorker): if lm_head is None: return _eager("no target lm_head") + # A gate-admitted quantized head is capture-safe: the target's own + # logits path already runs the same kernel under CUDA graphs. + head_supported = is_dense_head_weight( + getattr(lm_head, "weight", None) + ) or should_apply_lm_head_quant_method( + lm_head, getattr(lm_head, "quant_method", None) + ) + if self.selector is not None: # compute_candidates needs the target lm_head attached before capture. - # A gate-admitted quantized head is capture-safe: the target's own - # logits path already runs the same kernel under CUDA graphs. - if not is_dense_head_weight( - getattr(lm_head, "weight", None) - ) and not should_apply_lm_head_quant_method( - lm_head, getattr(lm_head, "quant_method", None) - ): + if not head_supported: return _eager("unsupported quantized lm_head") self.draft_model.lm_head = lm_head if self.model_runner.tp_rank == 0: @@ -829,6 +845,17 @@ class DFlashWorkerV2(BaseSpecWorker): device=self.device, sampling_enabled=self._selector_sampling_enabled, ) + if self.lilicorr is not None: + if not head_supported: + return _eager("unsupported quantized lm_head") + return build_lilicorr_draft_sampler( + sampling_enabled=self._lilicorr_sampling_enabled, + head=self.lilicorr, + draft_model=self.draft_model, + embed_tokens=target_input_embeddings(target_model), + lm_head=lm_head, + block_size=self.block_size, + ) if not hasattr(lm_head, "weight"): return _eager("quantized lm_head has no dense weight") if not is_dense_head_weight(lm_head.weight): @@ -1736,6 +1763,7 @@ class DFlashWorkerV2(BaseSpecWorker): positions: torch.Tensor, cache_loc_2d: Optional[torch.Tensor] = None, commit_lens: Optional[torch.Tensor] = None, + extend_lens: Optional[torch.Tensor] = None, ) -> None: """Materialize target context features into the draft KV cache at explicit slots. @@ -1835,6 +1863,13 @@ class DFlashWorkerV2(BaseSpecWorker): ), ): ctx_hidden = self.draft_model.project_target_hidden(target_hidden) + if self.lilicorr is not None: + self._lilicorr_anchor = publish_anchor( + draft_sampler=self._draft_sampler, + ctx_hidden=ctx_hidden, + extend_lens=extend_lens, + commit_lens=commit_lens, + ) if cache_loc_2d is not None: bs = int(commit_lens.shape[0]) @@ -2186,6 +2221,10 @@ class DFlashWorkerV2(BaseSpecWorker): self._warned_sampling_fallback = True return + # The sampled head hands verify the q it drew from, so no fallback is needed. + if self.lilicorr is not None and self._lilicorr_sampling_enabled: + return + if ( not is_dflash_sampling_verify_available() and not self._warned_sampling_fallback @@ -2283,6 +2322,7 @@ class DFlashWorkerV2(BaseSpecWorker): target_hidden=logits_output.hidden_states, cache_loc=batch.out_cache_loc, positions=positions, + extend_lens=ctx_lens, ) # Avoid copying large hidden-state buffers to CPU in overlap scheduling. @@ -2517,7 +2557,7 @@ class DFlashWorkerV2(BaseSpecWorker): global_num_token_non_padded_cpu=bs * block_size, ) - if self.selector is not None: + if self.selector is not None or self.lilicorr is not None: self._selector_sample = None if self._draft_sampler is not None: # Consumed by the in-graph sample; must be staged before the replay. @@ -2588,6 +2628,15 @@ class DFlashWorkerV2(BaseSpecWorker): self._draft_sampler.candidate_out[:bs], self._draft_sampler.q_out[:bs], ) + elif ( + self.lilicorr is not None + and self._lilicorr_sampling_enabled + and not _is_all_greedy(batch.sampling_info) + ): + self._selector_sample = ( + self._draft_sampler.candidate_out[:bs], + self._draft_sampler.q_out[:bs], + ) elif self.selector is not None: with self.draft_tp_context( self.draft_model_runner.tp_group, @@ -2600,6 +2649,40 @@ class DFlashWorkerV2(BaseSpecWorker): anchor_token_ids=block_ids[:, 0], sampling_info=batch.sampling_info, ) + elif self.lilicorr is not None: + if ( + self._lilicorr_sampling_enabled + and not self._warned_lilicorr_eager + and self.model_runner.tp_rank == 0 + ): + logger.warning( + "LiLiCorr sampled draft ran the eager head on a decode step " + "(draft cuda graph unavailable for batch size %d; captured " + "buckets do not cover it, or graphs are disabled). Acceptance is " + "unaffected but expect a large throughput regression, and do not " + "compare tokens/s from this run against a folded one.", + bs, + ) + self._warned_lilicorr_eager = True + draft_hidden = draft_logits_output.hidden_states + if draft_hidden is None: + raise RuntimeError("DFLASH draft model returned no hidden states.") + with self.draft_tp_context(self.draft_model_runner.tp_group): + draft_next, lilicorr_candidate_ids, lilicorr_q_rows = ( + propose_lilicorr_block( + head=self.lilicorr, + draft_hidden=draft_hidden.view(bs, int(self.block_size), -1), + lm_head=lm_head, + embed_tokens=target_input_embeddings( + self.target_worker.model_runner.model + ), + anchor=self._lilicorr_anchor, + sampling_info=batch.sampling_info, + sampling_enabled=self._lilicorr_sampling_enabled, + ) + ) + if lilicorr_q_rows is not None and not _is_all_greedy(batch.sampling_info): + self._selector_sample = (lilicorr_candidate_ids, lilicorr_q_rows) else: draft_hidden = draft_logits_output.hidden_states if draft_hidden is None: diff --git a/python/sglang/srt/speculative/lilicorr_utils.py b/python/sglang/srt/speculative/lilicorr_utils.py new file mode 100644 index 0000000000..cd2ac143d8 --- /dev/null +++ b/python/sglang/srt/speculative/lilicorr_utils.py @@ -0,0 +1,416 @@ +from __future__ import annotations + +import logging +from typing import Any, Optional, Tuple + +import msgspec +import torch + +from sglang.kernels.ops.speculative.lilicorr import MAX_FUSED_CANDIDATE_TOPK +from sglang.srt.environ import envs +from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding +from sglang.srt.models.dflash import candidate_topk +from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.speculative.dflash_utils import _get_dflash_config +from sglang.srt.speculative.dspark_components.dspark_draft import resolve_greedy_mask + +logger = logging.getLogger(__name__) + + +def resolve_sampling_enabled(*, device_supported: bool) -> bool: + requested = envs.SGLANG_ENABLE_LILICORR_SAMPLING.get() + if envs.SGLANG_LILICORR_REQUIRE_SAMPLING.get() and not requested: + raise RuntimeError( + "SGLANG_LILICORR_REQUIRE_SAMPLING is set but SGLANG_ENABLE_LILICORR_SAMPLING " + "is not; this process would serve the greedy commit under a sampled arm's name." + ) + logger.info("LiLiCorr draft commit: %s", "sampled" if requested else "greedy") + return requested and device_supported + + +class LiLiCorrConfig(msgspec.Struct, frozen=True): + candidate_topk: int + hidden_size: int + num_layers: int + num_heads: int + mlp_ratio: float + factor_dim: int + vector_eps: float + logit_scale: float + + def resolve_hidden_size(self, *, model_hidden_size: int) -> int: + # 0 means "as wide as the draft", recorded for a head with no token_proj. + return int(self.hidden_size) if self.hidden_size else int(model_hidden_size) + + +def _parse_lilicorr_config(dflash_cfg: dict) -> Optional[LiLiCorrConfig]: + if not any(key.startswith("lilicorr_") for key in dflash_cfg): + return None + enabled = dflash_cfg.get("lilicorr_enabled") + if enabled is not None and not bool(enabled): + return None + + def required(key: str, cast, *, positive: bool = True): + # No defaults: logit_scale and vector_eps change no shape, so load cannot catch them. + full_key = f"lilicorr_{key}" + if full_key not in dflash_cfg: + raise ValueError( + f"DFLASH dflash_config.{full_key} is required to rebuild the LiLiCorr " + "head, and the checkpoint does not carry it." + ) + try: + value = cast(dflash_cfg[full_key]) + except Exception as e: + raise ValueError( + f"Invalid dflash_config.{full_key}={dflash_cfg[full_key]!r}." + ) from e + if positive and value <= 0: + raise ValueError(f"dflash_config.{full_key} must be positive, got {value}.") + return value + + candidate_topk = required("candidate_topk", int) + if candidate_topk & (candidate_topk - 1): + raise ValueError( + f"dflash_config.lilicorr_candidate_topk must be a power of two, got " + f"{candidate_topk}: tl.arange requires a power-of-two extent." + ) + if candidate_topk > MAX_FUSED_CANDIDATE_TOPK: + raise ValueError( + f"dflash_config.lilicorr_candidate_topk={candidate_topk} exceeds the fused " + f"greedy commit's width of {MAX_FUSED_CANDIDATE_TOPK}, one Triton lane " + "group; a wider head would serve on the reference path." + ) + + return LiLiCorrConfig( + candidate_topk=candidate_topk, + hidden_size=required("hidden_size", int, positive=False), + num_layers=required("num_layers", int), + num_heads=required("num_heads", int), + mlp_ratio=required("mlp_ratio", float), + factor_dim=required("factor_dim", int), + vector_eps=required("vector_eps", float), + logit_scale=required("logit_scale", float), + ) + + +def parse_lilicorr_draft_config(*, draft_hf_config: Any) -> LiLiCorrConfig: + config = _parse_lilicorr_config(_get_dflash_config(draft_hf_config)) + if config is None: + raise ValueError( + "LiLiCorr requires the lilicorr_* geometry fields in dflash_config; a " + 'checkpoint declaring architectures=["LiLiCorrDraftModel"] without them ' + "cannot be rebuilt." + ) + return config + + +def reject_added_vocab(lm_head) -> None: + if not isinstance(lm_head, VocabParallelEmbedding): + return + if int(lm_head.shard_indices.num_added_elements) != 0: + raise NotImplementedError( + "LiLiCorr's candidate head does not support added vocabulary: those rows " + "sit past the padded base shard, so a contiguous top-k would skip them." + ) + + +def target_input_embeddings(target_model): + # Not the worker's _resolve_dflash_embedding_module, which returns the draft's own + # table for Nemotron-3.5: the head must embed with the table it was trained against. + embed = target_model.get_input_embeddings() + if embed is None: + raise RuntimeError("DFLASH target model exposes no input embeddings.") + return embed + + +def lilicorr_candidates( + *, hidden_states: torch.Tensor, lm_head, topk: int +) -> Tuple[torch.Tensor, torch.Tensor]: + # log_softmax(logits).topk(topk) over the FULL vocabulary: the head was trained on it. + ids, vals, lse = candidate_topk( + hidden_states, lm_head, int(topk), with_partition=True + ) + return vals.float() - lse.unsqueeze(-1), ids + + +def per_request_last_row( + *, + num_rows: int, + extend_lens: Optional[torch.Tensor], + commit_lens: Optional[torch.Tensor], +) -> Optional[torch.Tensor]: + # Verify's commit_lens index a [bs, block_size]-padded buffer (constant stride); + # prefill/extend's extend_lens index a packed ragged one. + if commit_lens is not None: + bs = int(commit_lens.shape[0]) + if bs == 0 or num_rows % bs != 0: + return None + stride = num_rows // bs + base = torch.arange(bs, device=commit_lens.device, dtype=torch.int64) * stride + return (base + commit_lens.to(torch.int64) - 1).clamp_min(0) + if extend_lens is None or extend_lens.numel() == 0: + return None + lens = extend_lens.to(torch.int64).flatten() + return (torch.cumsum(lens, dim=0) - 1).clamp_min(0) + + +def publish_anchor( + *, + draft_sampler, + ctx_hidden: torch.Tensor, + extend_lens: Optional[torch.Tensor] = None, + commit_lens: Optional[torch.Tensor] = None, +) -> Optional[torch.Tensor]: + ends = per_request_last_row( + num_rows=int(ctx_hidden.shape[0]), + extend_lens=extend_lens, + commit_lens=commit_lens, + ) + anchor = None if ends is None else ctx_hidden.index_select(0, ends) + if draft_sampler is not None: + draft_sampler.set_anchor(anchor, 0 if anchor is None else int(anchor.shape[0])) + return anchor + + +def propose_lilicorr_block( + *, + head, + draft_hidden: torch.Tensor, + lm_head, + embed_tokens, + anchor: Optional[torch.Tensor], + sampling_info=None, + sampling_enabled: bool = False, +) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: + bs, block_size, hidden_size = draft_hidden.shape + slots = block_size - 1 + pass_hidden = draft_hidden[:, 1:, :] + + log_probs, candidate_tokens = lilicorr_candidates( + hidden_states=pass_hidden.reshape(bs * slots, hidden_size), + lm_head=lm_head, + topk=int(head.candidate_topk), + ) + candidate_tokens = candidate_tokens.view(bs, slots, int(head.candidate_topk)) + + feat = int(head.context_proj.in_features) + if anchor is None or int(anchor.shape[0]) != bs: + # The batch resized since the anchor write; score invalid rather than misaligned. + anchor_hidden = torch.zeros( + (bs, feat), device=draft_hidden.device, dtype=draft_hidden.dtype + ) + anchor_valid = torch.zeros((bs,), dtype=torch.bool, device=draft_hidden.device) + else: + anchor_hidden = anchor.view(bs, feat) + anchor_valid = torch.ones((bs,), dtype=torch.bool, device=draft_hidden.device) + + # Outside the head because on the target model it may be a TP-sharded collective. + common = dict( + token_embeddings=embed_tokens(candidate_tokens).detach(), + candidate_tokens=candidate_tokens, + candidate_log_probs=log_probs.view(bs, slots, int(head.candidate_topk)), + pass_hidden=pass_hidden, + anchor_hidden=anchor_hidden, + anchor_valid=anchor_valid, + ) + device = draft_hidden.device + if sampling_enabled and sampling_info is not None: + # A greedy row's temperature may be 0, and it divides before greedy_mask applies. + temperatures = sampling_info.temperatures.view(-1)[:bs].float().clamp_min(1e-5) + else: + temperatures = torch.ones(bs, dtype=torch.float32, device=device) + greedy_mask = ( + resolve_greedy_mask(bs=bs, sampling_info=sampling_info, device=device) + if sampling_enabled + else torch.ones(bs, dtype=torch.bool, device=device) + ) + selected, q_rows = head.select_with_proposal( + uniforms=torch.rand(bs, slots, dtype=torch.float32, device=device), + temperatures=temperatures, + greedy_mask=greedy_mask, + **common, + ) + if not sampling_enabled: + return selected.to(torch.long), None, None + return selected.to(torch.long), candidate_tokens, q_rows + + +class LiLiCorrDraftSampler: + def __init__( + self, + *, + head, + embed_tokens, + lm_head, + block_size: int, + max_bs: int, + anchor_features: int, + sampling_enabled: bool, + ) -> None: + self.head = head + self.sampling_enabled = bool(sampling_enabled) + self.embed_tokens = embed_tokens + self.lm_head = lm_head + self.block_size = int(block_size) + self.slots = self.block_size - 1 + self.topk = int(head.candidate_topk) + self.max_bs = int(max_bs) + + # From the head, not from lm_head.weight: a quantized head's weight is packed, + # so its dtype is not the dtype these buffers carry. + anchor_row = next(head.parameters()) + device, dtype = anchor_row.device, anchor_row.dtype + max_rows = self.max_bs * self.slots + + self.out = torch.empty((max_rows,), dtype=torch.int64, device=device) + self.anchor = torch.zeros( + (self.max_bs, int(anchor_features)), dtype=dtype, device=device + ) + self.anchor_valid = torch.zeros((self.max_bs,), dtype=torch.bool, device=device) + self.token_table = head.build_token_table(embed_tokens) + + self.temperatures = torch.ones( + (self.max_bs,), dtype=torch.float32, device=device + ) + self.greedy_mask = torch.ones((self.max_bs,), dtype=torch.bool, device=device) + self.uniforms = torch.zeros( + (self.max_bs, self.slots), dtype=torch.float32, device=device + ) + self.q_out = torch.empty( + (self.max_bs, self.slots, self.topk), dtype=torch.float32, device=device + ) + self.candidate_out = torch.empty( + (self.max_bs, self.slots, self.topk), dtype=torch.int64, device=device + ) + + def set_anchor(self, rows: Optional[torch.Tensor], bs: int) -> None: + # Rows past bs are zeroed, not left stale: the graph runs at the bucket size. + # Filter/merge can hand a row a neighbour's anchor, costing acceptance only. + count = int(bs) + if rows is None or int(rows.shape[0]) != count or count > self.max_bs: + count = 0 + if count: + self.anchor[:count].copy_(rows) + self.anchor_valid[:count].fill_(True) + if count < self.max_bs: + self.anchor[count:].zero_() + self.anchor_valid[count:].fill_(False) + + def stage_sampling_params(self, *, bs: int, sampling_info) -> None: + if not self.sampling_enabled: + return + if sampling_info is None: + self.temperatures[:bs].fill_(1.0) + self.greedy_mask[:bs].fill_(True) + return + torch.clamp( + sampling_info.temperatures.view(-1)[:bs].to(torch.float32), + min=1e-5, + out=self.temperatures[:bs], + ) + self.greedy_mask[:bs].copy_( + resolve_greedy_mask( + bs=bs, sampling_info=sampling_info, device=self.greedy_mask.device + ) + ) + + def __call__(self, hidden_states: torch.Tensor, input_ids=None) -> None: + del input_ids + bs = hidden_states.shape[0] // self.block_size + rows = bs * self.slots + + if bs > self.max_bs: + raise RuntimeError( + f"LiLiCorrDraftSampler was built for max_bs={self.max_bs} but the draft " + f"graph replayed at bs={bs}; its static buffers are undersized." + ) + hidden_size = hidden_states.shape[-1] + pass_hidden = hidden_states.view(bs, self.block_size, hidden_size)[:, 1:, :] + log_probs, candidate_tokens = lilicorr_candidates( + hidden_states=pass_hidden.reshape(rows, hidden_size), + lm_head=self.lm_head, + topk=self.topk, + ) + candidate_tokens = candidate_tokens.view(bs, self.slots, self.topk) + + pre_projected = self.token_table is not None + token_embeddings = ( + self.token_table[candidate_tokens] + if pre_projected + else self.embed_tokens(candidate_tokens).detach() + ) + common = dict( + token_embeddings=token_embeddings, + candidate_tokens=candidate_tokens, + candidate_log_probs=log_probs.view(bs, self.slots, self.topk), + pass_hidden=pass_hidden, + anchor_hidden=self.anchor[:bs], + anchor_valid=self.anchor_valid[:bs], + already_projected=pre_projected, + ) + if self.sampling_enabled: + self.uniforms[:bs].uniform_() + selected, q_rows = self.head.select_with_proposal( + uniforms=self.uniforms[:bs], + temperatures=self.temperatures[:bs], + greedy_mask=self.greedy_mask[:bs], + **common, + ) + if self.sampling_enabled: + self.q_out[:bs].copy_(q_rows) + self.candidate_out[:bs].copy_(candidate_tokens) + self.out[:rows].copy_(selected.reshape(-1).to(torch.int64)) + + +def draft_graph_batch_sizes() -> list[int]: + return sorted( + {int(bs) for bs in get_exec().graph.cuda_graph_config.decode.bs if bs > 0} + ) + + +def build_lilicorr_draft_sampler( + *, + head, + draft_model, + embed_tokens, + lm_head, + block_size: int, + sampling_enabled: bool = False, +) -> Optional[LiLiCorrDraftSampler]: + def eager(reason: str) -> None: + logger.warning( + "LiLiCorr head kept eager (reason=%s): a bring-up path, not a serving " + "configuration. Expect a large throughput regression and numbers that are " + "not comparable to any published row.", + reason, + ) + return None + + tp_group = get_parallel().tp_group + if int(tp_group.world_size) != 1: + return eager("tp>1") + batch_sizes = draft_graph_batch_sizes() + if not batch_sizes: + return eager("no draft graph batch sizes to size the static buffers from") + max_bs = batch_sizes[-1] + + reject_added_vocab(lm_head) + parameter = next(head.parameters()) + head.materialize_inference_buffers(parameter.device, parameter.dtype) + + sampler = LiLiCorrDraftSampler( + head=head, + embed_tokens=embed_tokens, + lm_head=lm_head, + block_size=int(block_size), + max_bs=int(max_bs), + anchor_features=int(draft_model.fc.out_features), + sampling_enabled=sampling_enabled, + ) + logger.info( + "LiLiCorr select folded into the draft cuda graph: max_bs=%d block_size=%d K=%d.", + max_bs, + int(block_size), + sampler.topk, + ) + return sampler diff --git a/test/registered/e2e/speculative/test_dflash_lilicorr.py b/test/registered/e2e/speculative/test_dflash_lilicorr.py new file mode 100644 index 0000000000..6904fbe6fd --- /dev/null +++ b/test/registered/e2e/speculative/test_dflash_lilicorr.py @@ -0,0 +1,73 @@ +import unittest +from contextlib import ExitStack + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_cuda_ci( + est_time=300, + stage="extra-b", + runner_config="4-gpu-h100", + disabled="no LiLiCorr draft checkpoint is published yet", +) + +# The setUpClass skip covers `/rerun-test` and direct runs, which ignore `disabled=`. +UNPUBLISHED = "" +TARGET_MODEL = "Qwen/Qwen3-8B" +DRAFT_MODEL = UNPUBLISHED + + +class TestLiLiCorrServer(CustomTestCase, GSM8KMixin, SpecDecodingMixin): + model = TARGET_MODEL + draft_model = DRAFT_MODEL + # The floors below were measured on fa3 only. + attention_backend = "fa3" + gsm8k_accuracy_thres = 0.85 + # 5.07 with the head against 4.17 for the head-free DFLASH control. + gsm8k_accept_length_thres = 4.6 + # One prompt at 2048 tokens: 8.19 with the head against 7.31 for the control. + accept_length_thres = 7.7 + bs_1_speed_thres = 600.0 + + @classmethod + def setUpClass(cls): + if cls.draft_model == UNPUBLISHED: + raise unittest.SkipTest( + "no LiLiCorr draft checkpoint is published yet; set DRAFT_MODEL and drop " + "the disabled= argument on register_cuda_ci in the same change" + ) + cls.base_url = DEFAULT_URL_FOR_TEST + with ExitStack() as stack: + stack.enter_context(envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True)) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--attention-backend", + cls.attention_backend, + "--speculative-algorithm", + "DFLASH", + "--speculative-draft-model-path", + cls.draft_model, + "--mem-fraction-static", + "0.7", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/speculative/test_lilicorr_cuda.py b/test/registered/kernels/ops/speculative/test_lilicorr_cuda.py new file mode 100644 index 0000000000..bbc7c7bedf --- /dev/null +++ b/test/registered/kernels/ops/speculative/test_lilicorr_cuda.py @@ -0,0 +1,139 @@ +import sys + +import pytest +import torch + +from sglang.kernels.ops.speculative.lilicorr import ( + _lattice_scores, + _selector_walk_torch, + _topk_lse_torch, + lilicorr_sample_path, + lilicorr_topk_lse, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="extra-a", runner_config="1-gpu-large") + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available(), reason="LiLiCorr Triton kernels require CUDA" +) + + +# 1025 and 2047 straddle the tile boundary; 3072 has fewer tiles than k. +@pytest.mark.parametrize("vocab", [1025, 2047, 3072, 151936]) +def test_tiled_topk_lse_is_exact_against_the_reference(vocab): + torch.manual_seed(0) + logits = torch.randn(6, vocab, device="cuda", dtype=torch.float32) + + vals, tokens, lse = lilicorr_topk_lse(logits, 8) + ref_vals, ref_tokens, ref_lse = _topk_lse_torch(logits.cpu(), 8) + + torch.testing.assert_close(vals.cpu(), ref_vals) + torch.testing.assert_close(tokens.cpu(), ref_tokens) + torch.testing.assert_close(lse.cpu(), ref_lse) + + +def test_tied_bf16_logits_return_a_valid_selection(): + torch.manual_seed(6) + logits = torch.randint(0, 64, (4, 8192), device="cuda").to(torch.bfloat16) + + vals, tokens, _ = lilicorr_topk_lse(logits, 8) + ref_vals, _, _ = _topk_lse_torch(logits.cpu(), 8) + + torch.testing.assert_close(vals.cpu(), ref_vals) + gathered = torch.gather(logits, 1, tokens).float() + torch.testing.assert_close(gathered.cpu(), vals.cpu()) + + +def _walk_reference( + log_start, log_pair, candidate_tokens, uniforms, temperatures, greedy_mask +): + return _selector_walk_torch( + candidate_ids=candidate_tokens.cpu(), + scores=_lattice_scores(log_start.cpu(), log_pair.cpu()), + uniforms=uniforms.cpu(), + temperatures=temperatures.cpu(), + greedy_mask=greedy_mask.cpu(), + ) + + +def _greedy_state(bs, slots, device): + return dict( + uniforms=torch.zeros(bs, slots, device=device), + temperatures=torch.ones(bs, device=device), + greedy_mask=torch.ones(bs, dtype=torch.bool, device=device), + ) + + +def _greedy(log_start, log_pair, candidate_tokens): + bs, slots, _ = candidate_tokens.shape + tokens, _ = lilicorr_sample_path( + log_start, + log_pair, + candidate_tokens, + **_greedy_state(bs, slots, candidate_tokens.device), + ) + return tokens + + +def _greedy_reference(log_start, log_pair, candidate_tokens): + bs, slots, _ = candidate_tokens.shape + tokens, _ = _walk_reference( + log_start, + log_pair, + candidate_tokens, + **_greedy_state(bs, slots, torch.device("cpu")), + ) + return tokens + + +@pytest.mark.parametrize("topk", [1, 8, 16]) +def test_greedy_path_matches_the_reference(topk): + torch.manual_seed(4) + bs, slots = 5, 15 + log_start = torch.randn(bs, topk, device="cuda") + log_pair = torch.randn(bs, slots - 1, topk, topk, device="cuda") + tokens = torch.randint(0, 151936, (bs, slots, topk), device="cuda") + + actual = _greedy(log_start, log_pair, tokens) + expected = _greedy_reference(log_start, log_pair, tokens) + + torch.testing.assert_close(actual.cpu(), expected) + + +def test_greedy_path_breaks_ties_toward_the_lower_candidate_on_device(): + log_start = torch.zeros(2, 8, device="cuda") + log_pair = torch.zeros(2, 14, 8, 8, device="cuda") + tokens = torch.arange(2 * 15 * 8, device="cuda").view(2, 15, 8) + + actual = _greedy(log_start, log_pair, tokens) + expected = _greedy_reference(log_start, log_pair, tokens) + + torch.testing.assert_close(actual.cpu(), expected) + assert actual[:, 0].cpu().equal(tokens[:, 0, 0].cpu()) + + +def _sampled_inputs(bs, slots, k, *, seed=0): + torch.manual_seed(seed) + return dict( + log_start=torch.randn(bs, k, device="cuda"), + log_pair=torch.randn(bs, slots - 1, k, k, device="cuda"), + candidate_tokens=torch.arange(bs * slots * k, device="cuda").view(bs, slots, k), + uniforms=torch.rand(bs, slots, device="cuda"), + temperatures=torch.rand(bs, device="cuda") + 0.5, + greedy_mask=torch.zeros(bs, dtype=torch.bool, device="cuda"), + ) + + +@pytest.mark.parametrize("k", [1, 2, 8, 16]) +def test_sampled_path_matches_the_reference(k): + a = _sampled_inputs(4, 5, k, seed=k) + tokens, q = lilicorr_sample_path(**a) + ref_tokens, ref_q = _walk_reference(**a) + + assert torch.equal(tokens.cpu(), ref_tokens) + torch.testing.assert_close(q.cpu(), ref_q, atol=1e-6, rtol=1e-6) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/spec/test_dflash_logits.py b/test/registered/unit/spec/test_dflash_logits.py index f635a7b7f2..079fb2fae3 100644 --- a/test/registered/unit/spec/test_dflash_logits.py +++ b/test/registered/unit/spec/test_dflash_logits.py @@ -272,6 +272,35 @@ def test_worker_folds_a_gate_admitted_quantized_selector_head(monkeypatch): assert worker.draft_model.lm_head is None +def test_lilicorr_sampling_falls_back_when_the_device_cannot_accept_it(monkeypatch): + from sglang.srt.speculative import dflash_worker_v2 as worker_mod + + warnings = [] + monkeypatch.setattr( + worker_mod.logger, "warning", lambda *args: warnings.append(args) + ) + monkeypatch.setattr( + worker_mod, "is_dflash_sampling_verify_available", lambda: False + ) + worker = SimpleNamespace( + selector=None, + lilicorr=object(), + _lilicorr_sampling_enabled=False, + _warned_sampling_fallback=False, + model_runner=SimpleNamespace(tp_rank=0), + ) + batch = SimpleNamespace(sampling_info=SimpleNamespace(is_all_greedy=False)) + + worker_mod.DFlashWorkerV2._validate_phase1_sampling_support(worker, batch) + assert worker._warned_sampling_fallback + assert len(warnings) == 1 + + worker._lilicorr_sampling_enabled = True + worker._warned_sampling_fallback = False + worker_mod.DFlashWorkerV2._validate_phase1_sampling_support(worker, batch) + assert len(warnings) == 1 + + def test_worker_warns_once_when_selector_sampling_is_disabled(monkeypatch): from sglang.srt.speculative import dflash_worker_v2 as worker_mod