[Spec] LiLiCorr: named head MLPs and quantized head linears (#42057)

Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
mrusanovsky
2026-10-05 02:43:59 -07:00
committed by GitHub
co-authored by Po-Han Huang
parent 284cda01fb
commit 26a1377c50
3 changed files with 160 additions and 54 deletions
+7 -6
View File
@@ -624,6 +624,8 @@ class DFlashDraftModel(nn.Module):
decoder_layer_cls = DFlashDecoderLayer
supports_fused_context_kv = True
# Layer prefixes and a quant_config-aware `fc`, for quantized checkpoints.
supports_quantization = False
def __init__(self, config, quant_config=None, prefix: str = "") -> None:
super().__init__()
@@ -640,6 +642,7 @@ class DFlashDraftModel(nn.Module):
self.candidate_selector: Optional[nn.Module] = None
self.lilicorr: Optional[nn.Module] = None
self.is_nemotron_35_draft = is_nemotron_35_draft_config(config)
self.quantizable = self.supports_quantization or self.is_nemotron_35_draft
self.embed_tokens: Optional[VocabParallelEmbedding] = None
if self.is_nemotron_35_draft:
embed_prefix = f"{prefix}.embed_tokens" if prefix else "embed_tokens"
@@ -670,7 +673,7 @@ class DFlashDraftModel(nn.Module):
quant_config=quant_config,
prefix=(
(f"{prefix}.layers.{i}" if prefix else f"layers.{i}")
if self.is_nemotron_35_draft
if self.quantizable
else ""
),
)
@@ -694,7 +697,7 @@ class DFlashDraftModel(nn.Module):
num_context_features = len(target_layer_ids)
self.num_context_features = int(num_context_features)
if self.is_nemotron_35_draft:
if self.quantizable:
fc_prefix = f"{prefix}.fc" if prefix else "fc"
self.fc = ReplicatedLinear(
self.num_context_features * hidden_size,
@@ -764,9 +767,7 @@ class DFlashDraftModel(nn.Module):
def project_target_hidden(self, target_hidden: torch.Tensor) -> torch.Tensor:
"""Project concatenated target-layer hidden states into draft hidden_size."""
expected = int(
self.fc.input_size if self.is_nemotron_35_draft else self.fc.in_features
)
expected = int(self.fc.input_size if self.quantizable else self.fc.in_features)
if target_hidden.ndim != 2 or int(target_hidden.shape[-1]) != expected:
raise ValueError(
"DFLASH target_hidden feature dim mismatch. "
@@ -777,7 +778,7 @@ class DFlashDraftModel(nn.Module):
"the draft checkpoint/config expects."
)
projected = self.fc(target_hidden)
if self.is_nemotron_35_draft:
if self.quantizable:
projected = projected[0]
return self.hidden_norm(projected)
+141 -39
View File
@@ -9,19 +9,71 @@ from torch import nn
from sglang.kernels.ops.speculative.lilicorr import lilicorr_sample_path
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import LinearBase, ReplicatedLinear
from sglang.srt.layers.quantization.fp4_utils import get_fp4_gemm_runner_backend
from sglang.srt.layers.quantization.modelopt_quant import (
ModelOptFp4LinearMethod,
ModelOptNvFp4A16LinearMethod,
)
from sglang.srt.models.dflash import DFlashDraftModel
from sglang.srt.runtime_context import get_parallel
from sglang.srt.speculative.lilicorr_utils import (
LiLiCorrConfig,
parse_lilicorr_draft_config,
)
from sglang.srt.utils import add_prefix
logger = logging.getLogger(__name__)
def _apply_2d(layer: ReplicatedLinear, x: torch.Tensor) -> torch.Tensor:
# Quantized linear methods take [tokens, features] input.
out, _ = layer(x.reshape(-1, x.shape[-1]))
return out.view(*x.shape[:-1], -1)
def _is_nvfp4_a16(module: nn.Module) -> bool:
return isinstance(module, LinearBase) and isinstance(
module.quant_method, ModelOptNvFp4A16LinearMethod
)
def _runs_fp4_marlin(module: nn.Module) -> bool:
return _is_nvfp4_a16(module) or (
isinstance(module, LinearBase)
and isinstance(module.quant_method, ModelOptFp4LinearMethod)
and get_fp4_gemm_runner_backend().is_marlin()
)
class LiLiCorrMLP(nn.Module):
def __init__(
self, in_size: int, mid_size: int, out_size: int, quant_config, prefix: str
) -> None:
super().__init__()
self.up_proj = ReplicatedLinear(
in_size,
mid_size,
quant_config=quant_config,
prefix=add_prefix("up_proj", prefix),
)
self.down_proj = ReplicatedLinear(
mid_size,
out_size,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return _apply_2d(self.down_proj, F.silu(_apply_2d(self.up_proj, x)))
class LiLiCorrLatticeAttention(nn.Module):
# nn.MultiheadAttention's parameter layout, run via SDPA since that module is not capturable.
def __init__(self, hidden_size: int, num_heads: int) -> None:
def __init__(
self, hidden_size: int, num_heads: int, quant_config=None, prefix: str = ""
) -> None:
super().__init__()
if hidden_size % num_heads != 0:
raise ValueError(
@@ -33,7 +85,12 @@ class LiLiCorrLatticeAttention(nn.Module):
self.head_dim = self.hidden_size // self.num_heads
self.in_proj_weight = nn.Parameter(torch.zeros(3 * hidden_size, hidden_size))
self.in_proj_bias = nn.Parameter(torch.zeros(3 * hidden_size))
self.out_proj = nn.Linear(hidden_size, hidden_size)
self.out_proj = ReplicatedLinear(
hidden_size,
hidden_size,
quant_config=quant_config,
prefix=add_prefix("out_proj", prefix),
)
def forward(
self, hidden_states: torch.Tensor, attention_bias: torch.Tensor
@@ -47,9 +104,8 @@ class LiLiCorrLatticeAttention(nn.Module):
v = v.view(shape).transpose(1, 2)
# attention_bias is [1, heads, L, L]; SDPA broadcasts it over the batch.
out = F.scaled_dot_product_attention(q, k, v, attn_mask=attention_bias)
return self.out_proj(
out.transpose(1, 2).reshape(bsz, seq_len, self.hidden_size)
)
out, _ = self.out_proj(out.transpose(1, 2).reshape(-1, self.hidden_size))
return out.view(bsz, seq_len, self.hidden_size)
class LiLiCorrLayer(nn.Module):
@@ -59,16 +115,21 @@ class LiLiCorrLayer(nn.Module):
num_heads: int,
mlp_ratio: float,
rms_norm_eps: float,
quant_config=None,
prefix: str = "",
) -> None:
super().__init__()
self.attn_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
self.attn = LiLiCorrLatticeAttention(hidden_size, num_heads)
self.attn = LiLiCorrLatticeAttention(
hidden_size, num_heads, quant_config, add_prefix("attn", prefix)
)
self.mlp_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
mlp_hidden_size = int(hidden_size * mlp_ratio)
self.mlp = nn.Sequential(
nn.Linear(hidden_size, mlp_hidden_size),
nn.SiLU(),
nn.Linear(mlp_hidden_size, hidden_size),
self.mlp = LiLiCorrMLP(
hidden_size,
int(hidden_size * mlp_ratio),
hidden_size,
quant_config,
add_prefix("mlp", prefix),
)
def forward(
@@ -91,6 +152,8 @@ class LiLiCorrHead(nn.Module):
block_size: int,
rms_norm_eps: float,
config: LiLiCorrConfig,
quant_config=None,
prefix: str = "",
) -> None:
super().__init__()
hidden_size = config.resolve_hidden_size(model_hidden_size=model_hidden_size)
@@ -106,16 +169,29 @@ class LiLiCorrHead(nn.Module):
self.logit_scale = float(config.logit_scale)
self.token_proj = (
nn.Identity()
None
if model_hidden_size == hidden_size
else nn.Linear(model_hidden_size, hidden_size)
else ReplicatedLinear(
model_hidden_size,
hidden_size,
quant_config=quant_config,
prefix=add_prefix("token_proj", prefix),
)
)
self.pass_hidden_proj = nn.Linear(model_hidden_size, hidden_size)
self.feature_mlp = nn.Sequential(
nn.LayerNorm(self.num_candidate_features),
nn.Linear(self.num_candidate_features, hidden_size),
nn.SiLU(),
nn.Linear(hidden_size, hidden_size),
self.pass_hidden_proj = ReplicatedLinear(
model_hidden_size,
hidden_size,
quant_config=quant_config,
prefix=add_prefix("pass_hidden_proj", prefix),
)
self.feature_norm = nn.LayerNorm(self.num_candidate_features)
# Five input features fit no quantized GEMM kernel.
self.feature_mlp = LiLiCorrMLP(
self.num_candidate_features,
hidden_size,
hidden_size,
None,
add_prefix("feature_mlp", prefix),
)
self.slot_embedding = nn.Parameter(
torch.zeros(1, 1, self.num_candidate_slots, 1, hidden_size)
@@ -127,7 +203,12 @@ class LiLiCorrHead(nn.Module):
torch.zeros(self.num_heads, 2 * self.block_size - 1)
)
self.same_slot_bias = nn.Parameter(torch.zeros(self.num_heads))
self.context_proj = nn.Linear(model_hidden_size, hidden_size)
self.context_proj = ReplicatedLinear(
model_hidden_size,
hidden_size,
quant_config=quant_config,
prefix=add_prefix("context_proj", prefix),
)
self.layers = nn.ModuleList(
[
LiLiCorrLayer(
@@ -135,16 +216,24 @@ class LiLiCorrHead(nn.Module):
num_heads=self.num_heads,
mlp_ratio=self.mlp_ratio,
rms_norm_eps=rms_norm_eps,
quant_config=quant_config,
prefix=add_prefix(f"layers.{i}", prefix),
)
for _ in range(int(config.num_layers))
for i in range(int(config.num_layers))
]
)
self.output_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
self.anchor_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
# Read as raw weights by materialize_inference_buffers, so never quantized.
self.factor_input_proj = nn.Linear(hidden_size * 3, hidden_size)
self.out_head = nn.Linear(hidden_size, self.factor_dim)
self.in_head = nn.Linear(hidden_size, self.factor_dim)
self.anchor_out_head = nn.Linear(hidden_size, self.factor_dim)
self.anchor_out_head = ReplicatedLinear(
hidden_size,
self.factor_dim,
quant_config=quant_config,
prefix=add_prefix("anchor_out_head", prefix),
)
self._attn_bias: Optional[torch.Tensor] = None
self._fused_edge_weight: Optional[torch.Tensor] = None
@@ -231,7 +320,7 @@ class LiLiCorrHead(nn.Module):
self, anchor_hidden: torch.Tensor, anchor_valid: torch.Tensor
) -> torch.Tensor:
# Branch-free so there is no host sync inside the captured region.
anchor = self.context_proj(anchor_hidden)
anchor = _apply_2d(self.context_proj, anchor_hidden)
return anchor * anchor_valid.unsqueeze(-1).to(anchor.dtype)
def score(
@@ -252,16 +341,16 @@ class LiLiCorrHead(nn.Module):
f"lattice carries {topk}; both are sized for the trained width."
)
proj_dtype = self.pass_hidden_proj.weight.dtype
proj_dtype = self.slot_embedding.dtype
if token_embeddings.dtype != proj_dtype:
token_embeddings = token_embeddings.to(proj_dtype)
if pass_hidden.dtype != proj_dtype:
pass_hidden = pass_hidden.to(proj_dtype)
token_states = (
token_embeddings if already_projected else self.token_proj(token_embeddings)
)
pass_states = self.pass_hidden_proj(pass_hidden).unsqueeze(-2)
token_states = token_embeddings
if not already_projected and self.token_proj is not None:
token_states = _apply_2d(self.token_proj, token_embeddings)
pass_states = _apply_2d(self.pass_hidden_proj, pass_hidden).unsqueeze(-2)
log_probs = candidate_log_probs.float()
features = torch.stack(
@@ -276,7 +365,7 @@ class LiLiCorrHead(nn.Module):
)
hidden_states = token_states + pass_states
hidden_states = hidden_states + self.feature_mlp(
features.to(dtype=token_states.dtype)
self.feature_norm(features.to(dtype=token_states.dtype))
)
hidden_states = hidden_states + self.slot_embedding[:, :, :n_slots]
hidden_states = hidden_states + self.rank_embedding
@@ -307,7 +396,7 @@ class LiLiCorrHead(nn.Module):
eps=self.vector_eps,
).unbind(-2)
anchor_out = F.normalize(
self.anchor_out_head(anchor_state), dim=-1, eps=self.vector_eps
_apply_2d(self.anchor_out_head, anchor_state), dim=-1, eps=self.vector_eps
)
start_scores = (anchor_out[:, :, None, :] * in_vec[:, :, 0, :, :]).sum(dim=-1)
@@ -358,23 +447,26 @@ class LiLiCorrHead(nn.Module):
@torch.no_grad()
def build_token_table(self, embed_tokens: nn.Module) -> Optional[torch.Tensor]:
if isinstance(self.token_proj, nn.Identity):
if self.token_proj is None:
return None
weight = embed_tokens.weight
if int(self.token_proj.weight.shape[1]) != int(weight.shape[1]):
if self.token_proj.input_size != int(weight.shape[1]):
return None
return F.linear(
weight.to(self.token_proj.weight.dtype),
self.token_proj.weight,
self.token_proj.bias,
).contiguous()
table, _ = self.token_proj(weight.to(self.slot_embedding.dtype))
return table.contiguous()
def check_head_weight_coverage(head: LiLiCorrHead, seen: set) -> None:
# The base loader silently drops unresolved weights, which would only show
# up as a lower acceptance length.
expected = {f"lilicorr.{name}" for name, _ in head.named_parameters()}
missing = sorted(expected - seen)
# NVFP4 W4A16 registers an input_scale it never reads; checkpoints omit it.
optional = {
f"lilicorr.{name}.input_scale"
for name, module in head.named_modules()
if _is_nvfp4_a16(module)
}
missing = sorted(expected - optional - seen)
if missing:
raise ValueError(
f"LiLiCorr checkpoint is missing {len(missing)} head parameters "
@@ -435,6 +527,8 @@ def check_conv_weight_coverage(model: DFlashDraftModel, seen: set) -> None:
class LiLiCorrDraftModel(DFlashDraftModel):
supports_quantization = True
def __init__(self, config, quant_config=None, prefix: str = "") -> None:
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
self.lilicorr = LiLiCorrHead(
@@ -442,7 +536,15 @@ class LiLiCorrDraftModel(DFlashDraftModel):
block_size=int(self.block_size),
rms_norm_eps=self.rms_norm_eps,
config=parse_lilicorr_draft_config(draft_hf_config=config),
quant_config=quant_config,
prefix=add_prefix("lilicorr", prefix),
)
# Every TP rank runs the replicated head, so its draft must match bit for bit.
# Marlin's atomic-add reduce at the head's shapes does not.
if get_parallel().tp_size > 1 and any(
_runs_fp4_marlin(m) for m in self.lilicorr.modules()
):
raise ValueError("A LiLiCorr head on NVFP4 Marlin requires tp_size 1.")
def set_block_size(self, block_size: int) -> None:
super().set_block_size(block_size)
@@ -466,7 +568,7 @@ class LiLiCorrDraftModel(DFlashDraftModel):
check_head_weight_coverage(self.lilicorr, seen)
check_conv_weight_coverage(self, seen_conv)
parameter = next(self.lilicorr.parameters())
parameter = self.lilicorr.slot_embedding
self.lilicorr.materialize_inference_buffers(parameter.device, parameter.dtype)
@@ -7,10 +7,11 @@ import msgspec
import torch
from sglang.kernels.ops.speculative.lilicorr import MAX_FUSED_CANDIDATE_TOPK
from sglang.srt.distributed.communication_op import tensor_model_parallel_all_gather
from sglang.srt.environ import envs
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
from sglang.srt.models.dflash import candidate_topk
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.runtime_context import get_exec
from sglang.srt.speculative.dflash_utils import _get_dflash_config
from sglang.srt.speculative.dspark_components.dspark_draft import resolve_greedy_mask
@@ -193,7 +194,7 @@ def propose_lilicorr_block(
)
candidate_tokens = candidate_tokens.view(bs, slots, int(head.candidate_topk))
feat = int(head.context_proj.in_features)
feat = int(head.context_proj.input_size)
if anchor is None or int(anchor.shape[0]) != bs:
# The batch resized since the anchor write; score invalid rather than misaligned.
anchor_hidden = torch.zeros(
@@ -258,7 +259,7 @@ class LiLiCorrDraftSampler:
# From the head, not from lm_head.weight: a quantized head's weight is packed,
# so its dtype is not the dtype these buffers carry.
anchor_row = next(head.parameters())
anchor_row = head.slot_embedding
device, dtype = anchor_row.device, anchor_row.dtype
max_rows = self.max_bs * self.slots
@@ -267,7 +268,12 @@ class LiLiCorrDraftSampler:
(self.max_bs, int(anchor_features)), dtype=dtype, device=device
)
self.anchor_valid = torch.zeros((self.max_bs,), dtype=torch.bool, device=device)
self.token_table = head.build_token_table(embed_tokens)
# Candidates carry global ids, so every rank holds the whole table.
token_table = head.build_token_table(embed_tokens)
if token_table is not None and getattr(embed_tokens, "tp_size", 1) > 1:
token_table = tensor_model_parallel_all_gather(token_table, dim=0)
assert token_table.shape[0] == embed_tokens.org_vocab_size_padded
self.token_table = token_table
self.temperatures = torch.ones(
(self.max_bs,), dtype=torch.float32, device=device
@@ -386,16 +392,13 @@ def build_lilicorr_draft_sampler(
)
return None
tp_group = get_parallel().tp_group
if int(tp_group.world_size) != 1:
return eager("tp>1")
batch_sizes = draft_graph_batch_sizes()
if not batch_sizes:
return eager("no draft graph batch sizes to size the static buffers from")
max_bs = batch_sizes[-1]
reject_added_vocab(lm_head)
parameter = next(head.parameters())
parameter = head.slot_embedding
head.materialize_inference_buffers(parameter.device, parameter.dtype)
sampler = LiLiCorrDraftSampler(
@@ -404,7 +407,7 @@ def build_lilicorr_draft_sampler(
lm_head=lm_head,
block_size=int(block_size),
max_bs=int(max_bs),
anchor_features=int(draft_model.fc.out_features),
anchor_features=int(draft_model.fc.output_size),
sampling_enabled=sampling_enabled,
)
logger.info(