[Spec] LiLiCorr: named head MLPs and quantized head linears (#42057)
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
Po-Han Huang
parent
284cda01fb
commit
26a1377c50
@@ -624,6 +624,8 @@ class DFlashDraftModel(nn.Module):
|
||||
|
||||
decoder_layer_cls = DFlashDecoderLayer
|
||||
supports_fused_context_kv = True
|
||||
# Layer prefixes and a quant_config-aware `fc`, for quantized checkpoints.
|
||||
supports_quantization = False
|
||||
|
||||
def __init__(self, config, quant_config=None, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
@@ -640,6 +642,7 @@ class DFlashDraftModel(nn.Module):
|
||||
self.candidate_selector: Optional[nn.Module] = None
|
||||
self.lilicorr: Optional[nn.Module] = None
|
||||
self.is_nemotron_35_draft = is_nemotron_35_draft_config(config)
|
||||
self.quantizable = self.supports_quantization or self.is_nemotron_35_draft
|
||||
self.embed_tokens: Optional[VocabParallelEmbedding] = None
|
||||
if self.is_nemotron_35_draft:
|
||||
embed_prefix = f"{prefix}.embed_tokens" if prefix else "embed_tokens"
|
||||
@@ -670,7 +673,7 @@ class DFlashDraftModel(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=(
|
||||
(f"{prefix}.layers.{i}" if prefix else f"layers.{i}")
|
||||
if self.is_nemotron_35_draft
|
||||
if self.quantizable
|
||||
else ""
|
||||
),
|
||||
)
|
||||
@@ -694,7 +697,7 @@ class DFlashDraftModel(nn.Module):
|
||||
num_context_features = len(target_layer_ids)
|
||||
|
||||
self.num_context_features = int(num_context_features)
|
||||
if self.is_nemotron_35_draft:
|
||||
if self.quantizable:
|
||||
fc_prefix = f"{prefix}.fc" if prefix else "fc"
|
||||
self.fc = ReplicatedLinear(
|
||||
self.num_context_features * hidden_size,
|
||||
@@ -764,9 +767,7 @@ class DFlashDraftModel(nn.Module):
|
||||
|
||||
def project_target_hidden(self, target_hidden: torch.Tensor) -> torch.Tensor:
|
||||
"""Project concatenated target-layer hidden states into draft hidden_size."""
|
||||
expected = int(
|
||||
self.fc.input_size if self.is_nemotron_35_draft else self.fc.in_features
|
||||
)
|
||||
expected = int(self.fc.input_size if self.quantizable else self.fc.in_features)
|
||||
if target_hidden.ndim != 2 or int(target_hidden.shape[-1]) != expected:
|
||||
raise ValueError(
|
||||
"DFLASH target_hidden feature dim mismatch. "
|
||||
@@ -777,7 +778,7 @@ class DFlashDraftModel(nn.Module):
|
||||
"the draft checkpoint/config expects."
|
||||
)
|
||||
projected = self.fc(target_hidden)
|
||||
if self.is_nemotron_35_draft:
|
||||
if self.quantizable:
|
||||
projected = projected[0]
|
||||
return self.hidden_norm(projected)
|
||||
|
||||
|
||||
@@ -9,19 +9,71 @@ from torch import nn
|
||||
|
||||
from sglang.kernels.ops.speculative.lilicorr import lilicorr_sample_path
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import LinearBase, ReplicatedLinear
|
||||
from sglang.srt.layers.quantization.fp4_utils import get_fp4_gemm_runner_backend
|
||||
from sglang.srt.layers.quantization.modelopt_quant import (
|
||||
ModelOptFp4LinearMethod,
|
||||
ModelOptNvFp4A16LinearMethod,
|
||||
)
|
||||
from sglang.srt.models.dflash import DFlashDraftModel
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.speculative.lilicorr_utils import (
|
||||
LiLiCorrConfig,
|
||||
parse_lilicorr_draft_config,
|
||||
)
|
||||
from sglang.srt.utils import add_prefix
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _apply_2d(layer: ReplicatedLinear, x: torch.Tensor) -> torch.Tensor:
|
||||
# Quantized linear methods take [tokens, features] input.
|
||||
out, _ = layer(x.reshape(-1, x.shape[-1]))
|
||||
return out.view(*x.shape[:-1], -1)
|
||||
|
||||
|
||||
def _is_nvfp4_a16(module: nn.Module) -> bool:
|
||||
return isinstance(module, LinearBase) and isinstance(
|
||||
module.quant_method, ModelOptNvFp4A16LinearMethod
|
||||
)
|
||||
|
||||
|
||||
def _runs_fp4_marlin(module: nn.Module) -> bool:
|
||||
return _is_nvfp4_a16(module) or (
|
||||
isinstance(module, LinearBase)
|
||||
and isinstance(module.quant_method, ModelOptFp4LinearMethod)
|
||||
and get_fp4_gemm_runner_backend().is_marlin()
|
||||
)
|
||||
|
||||
|
||||
class LiLiCorrMLP(nn.Module):
|
||||
def __init__(
|
||||
self, in_size: int, mid_size: int, out_size: int, quant_config, prefix: str
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.up_proj = ReplicatedLinear(
|
||||
in_size,
|
||||
mid_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("up_proj", prefix),
|
||||
)
|
||||
self.down_proj = ReplicatedLinear(
|
||||
mid_size,
|
||||
out_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return _apply_2d(self.down_proj, F.silu(_apply_2d(self.up_proj, x)))
|
||||
|
||||
|
||||
class LiLiCorrLatticeAttention(nn.Module):
|
||||
# nn.MultiheadAttention's parameter layout, run via SDPA since that module is not capturable.
|
||||
|
||||
def __init__(self, hidden_size: int, num_heads: int) -> None:
|
||||
def __init__(
|
||||
self, hidden_size: int, num_heads: int, quant_config=None, prefix: str = ""
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if hidden_size % num_heads != 0:
|
||||
raise ValueError(
|
||||
@@ -33,7 +85,12 @@ class LiLiCorrLatticeAttention(nn.Module):
|
||||
self.head_dim = self.hidden_size // self.num_heads
|
||||
self.in_proj_weight = nn.Parameter(torch.zeros(3 * hidden_size, hidden_size))
|
||||
self.in_proj_bias = nn.Parameter(torch.zeros(3 * hidden_size))
|
||||
self.out_proj = nn.Linear(hidden_size, hidden_size)
|
||||
self.out_proj = ReplicatedLinear(
|
||||
hidden_size,
|
||||
hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("out_proj", prefix),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, attention_bias: torch.Tensor
|
||||
@@ -47,9 +104,8 @@ class LiLiCorrLatticeAttention(nn.Module):
|
||||
v = v.view(shape).transpose(1, 2)
|
||||
# attention_bias is [1, heads, L, L]; SDPA broadcasts it over the batch.
|
||||
out = F.scaled_dot_product_attention(q, k, v, attn_mask=attention_bias)
|
||||
return self.out_proj(
|
||||
out.transpose(1, 2).reshape(bsz, seq_len, self.hidden_size)
|
||||
)
|
||||
out, _ = self.out_proj(out.transpose(1, 2).reshape(-1, self.hidden_size))
|
||||
return out.view(bsz, seq_len, self.hidden_size)
|
||||
|
||||
|
||||
class LiLiCorrLayer(nn.Module):
|
||||
@@ -59,16 +115,21 @@ class LiLiCorrLayer(nn.Module):
|
||||
num_heads: int,
|
||||
mlp_ratio: float,
|
||||
rms_norm_eps: float,
|
||||
quant_config=None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.attn_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
|
||||
self.attn = LiLiCorrLatticeAttention(hidden_size, num_heads)
|
||||
self.attn = LiLiCorrLatticeAttention(
|
||||
hidden_size, num_heads, quant_config, add_prefix("attn", prefix)
|
||||
)
|
||||
self.mlp_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
|
||||
mlp_hidden_size = int(hidden_size * mlp_ratio)
|
||||
self.mlp = nn.Sequential(
|
||||
nn.Linear(hidden_size, mlp_hidden_size),
|
||||
nn.SiLU(),
|
||||
nn.Linear(mlp_hidden_size, hidden_size),
|
||||
self.mlp = LiLiCorrMLP(
|
||||
hidden_size,
|
||||
int(hidden_size * mlp_ratio),
|
||||
hidden_size,
|
||||
quant_config,
|
||||
add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -91,6 +152,8 @@ class LiLiCorrHead(nn.Module):
|
||||
block_size: int,
|
||||
rms_norm_eps: float,
|
||||
config: LiLiCorrConfig,
|
||||
quant_config=None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
hidden_size = config.resolve_hidden_size(model_hidden_size=model_hidden_size)
|
||||
@@ -106,16 +169,29 @@ class LiLiCorrHead(nn.Module):
|
||||
self.logit_scale = float(config.logit_scale)
|
||||
|
||||
self.token_proj = (
|
||||
nn.Identity()
|
||||
None
|
||||
if model_hidden_size == hidden_size
|
||||
else nn.Linear(model_hidden_size, hidden_size)
|
||||
else ReplicatedLinear(
|
||||
model_hidden_size,
|
||||
hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("token_proj", prefix),
|
||||
)
|
||||
)
|
||||
self.pass_hidden_proj = nn.Linear(model_hidden_size, hidden_size)
|
||||
self.feature_mlp = nn.Sequential(
|
||||
nn.LayerNorm(self.num_candidate_features),
|
||||
nn.Linear(self.num_candidate_features, hidden_size),
|
||||
nn.SiLU(),
|
||||
nn.Linear(hidden_size, hidden_size),
|
||||
self.pass_hidden_proj = ReplicatedLinear(
|
||||
model_hidden_size,
|
||||
hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("pass_hidden_proj", prefix),
|
||||
)
|
||||
self.feature_norm = nn.LayerNorm(self.num_candidate_features)
|
||||
# Five input features fit no quantized GEMM kernel.
|
||||
self.feature_mlp = LiLiCorrMLP(
|
||||
self.num_candidate_features,
|
||||
hidden_size,
|
||||
hidden_size,
|
||||
None,
|
||||
add_prefix("feature_mlp", prefix),
|
||||
)
|
||||
self.slot_embedding = nn.Parameter(
|
||||
torch.zeros(1, 1, self.num_candidate_slots, 1, hidden_size)
|
||||
@@ -127,7 +203,12 @@ class LiLiCorrHead(nn.Module):
|
||||
torch.zeros(self.num_heads, 2 * self.block_size - 1)
|
||||
)
|
||||
self.same_slot_bias = nn.Parameter(torch.zeros(self.num_heads))
|
||||
self.context_proj = nn.Linear(model_hidden_size, hidden_size)
|
||||
self.context_proj = ReplicatedLinear(
|
||||
model_hidden_size,
|
||||
hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("context_proj", prefix),
|
||||
)
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
LiLiCorrLayer(
|
||||
@@ -135,16 +216,24 @@ class LiLiCorrHead(nn.Module):
|
||||
num_heads=self.num_heads,
|
||||
mlp_ratio=self.mlp_ratio,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(f"layers.{i}", prefix),
|
||||
)
|
||||
for _ in range(int(config.num_layers))
|
||||
for i in range(int(config.num_layers))
|
||||
]
|
||||
)
|
||||
self.output_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
|
||||
self.anchor_norm = RMSNorm(hidden_size, eps=rms_norm_eps)
|
||||
# Read as raw weights by materialize_inference_buffers, so never quantized.
|
||||
self.factor_input_proj = nn.Linear(hidden_size * 3, hidden_size)
|
||||
self.out_head = nn.Linear(hidden_size, self.factor_dim)
|
||||
self.in_head = nn.Linear(hidden_size, self.factor_dim)
|
||||
self.anchor_out_head = nn.Linear(hidden_size, self.factor_dim)
|
||||
self.anchor_out_head = ReplicatedLinear(
|
||||
hidden_size,
|
||||
self.factor_dim,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("anchor_out_head", prefix),
|
||||
)
|
||||
|
||||
self._attn_bias: Optional[torch.Tensor] = None
|
||||
self._fused_edge_weight: Optional[torch.Tensor] = None
|
||||
@@ -231,7 +320,7 @@ class LiLiCorrHead(nn.Module):
|
||||
self, anchor_hidden: torch.Tensor, anchor_valid: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
# Branch-free so there is no host sync inside the captured region.
|
||||
anchor = self.context_proj(anchor_hidden)
|
||||
anchor = _apply_2d(self.context_proj, anchor_hidden)
|
||||
return anchor * anchor_valid.unsqueeze(-1).to(anchor.dtype)
|
||||
|
||||
def score(
|
||||
@@ -252,16 +341,16 @@ class LiLiCorrHead(nn.Module):
|
||||
f"lattice carries {topk}; both are sized for the trained width."
|
||||
)
|
||||
|
||||
proj_dtype = self.pass_hidden_proj.weight.dtype
|
||||
proj_dtype = self.slot_embedding.dtype
|
||||
if token_embeddings.dtype != proj_dtype:
|
||||
token_embeddings = token_embeddings.to(proj_dtype)
|
||||
if pass_hidden.dtype != proj_dtype:
|
||||
pass_hidden = pass_hidden.to(proj_dtype)
|
||||
|
||||
token_states = (
|
||||
token_embeddings if already_projected else self.token_proj(token_embeddings)
|
||||
)
|
||||
pass_states = self.pass_hidden_proj(pass_hidden).unsqueeze(-2)
|
||||
token_states = token_embeddings
|
||||
if not already_projected and self.token_proj is not None:
|
||||
token_states = _apply_2d(self.token_proj, token_embeddings)
|
||||
pass_states = _apply_2d(self.pass_hidden_proj, pass_hidden).unsqueeze(-2)
|
||||
|
||||
log_probs = candidate_log_probs.float()
|
||||
features = torch.stack(
|
||||
@@ -276,7 +365,7 @@ class LiLiCorrHead(nn.Module):
|
||||
)
|
||||
hidden_states = token_states + pass_states
|
||||
hidden_states = hidden_states + self.feature_mlp(
|
||||
features.to(dtype=token_states.dtype)
|
||||
self.feature_norm(features.to(dtype=token_states.dtype))
|
||||
)
|
||||
hidden_states = hidden_states + self.slot_embedding[:, :, :n_slots]
|
||||
hidden_states = hidden_states + self.rank_embedding
|
||||
@@ -307,7 +396,7 @@ class LiLiCorrHead(nn.Module):
|
||||
eps=self.vector_eps,
|
||||
).unbind(-2)
|
||||
anchor_out = F.normalize(
|
||||
self.anchor_out_head(anchor_state), dim=-1, eps=self.vector_eps
|
||||
_apply_2d(self.anchor_out_head, anchor_state), dim=-1, eps=self.vector_eps
|
||||
)
|
||||
|
||||
start_scores = (anchor_out[:, :, None, :] * in_vec[:, :, 0, :, :]).sum(dim=-1)
|
||||
@@ -358,23 +447,26 @@ class LiLiCorrHead(nn.Module):
|
||||
|
||||
@torch.no_grad()
|
||||
def build_token_table(self, embed_tokens: nn.Module) -> Optional[torch.Tensor]:
|
||||
if isinstance(self.token_proj, nn.Identity):
|
||||
if self.token_proj is None:
|
||||
return None
|
||||
weight = embed_tokens.weight
|
||||
if int(self.token_proj.weight.shape[1]) != int(weight.shape[1]):
|
||||
if self.token_proj.input_size != int(weight.shape[1]):
|
||||
return None
|
||||
return F.linear(
|
||||
weight.to(self.token_proj.weight.dtype),
|
||||
self.token_proj.weight,
|
||||
self.token_proj.bias,
|
||||
).contiguous()
|
||||
table, _ = self.token_proj(weight.to(self.slot_embedding.dtype))
|
||||
return table.contiguous()
|
||||
|
||||
|
||||
def check_head_weight_coverage(head: LiLiCorrHead, seen: set) -> None:
|
||||
# The base loader silently drops unresolved weights, which would only show
|
||||
# up as a lower acceptance length.
|
||||
expected = {f"lilicorr.{name}" for name, _ in head.named_parameters()}
|
||||
missing = sorted(expected - seen)
|
||||
# NVFP4 W4A16 registers an input_scale it never reads; checkpoints omit it.
|
||||
optional = {
|
||||
f"lilicorr.{name}.input_scale"
|
||||
for name, module in head.named_modules()
|
||||
if _is_nvfp4_a16(module)
|
||||
}
|
||||
missing = sorted(expected - optional - seen)
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"LiLiCorr checkpoint is missing {len(missing)} head parameters "
|
||||
@@ -435,6 +527,8 @@ def check_conv_weight_coverage(model: DFlashDraftModel, seen: set) -> None:
|
||||
|
||||
|
||||
class LiLiCorrDraftModel(DFlashDraftModel):
|
||||
supports_quantization = True
|
||||
|
||||
def __init__(self, config, quant_config=None, prefix: str = "") -> None:
|
||||
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
|
||||
self.lilicorr = LiLiCorrHead(
|
||||
@@ -442,7 +536,15 @@ class LiLiCorrDraftModel(DFlashDraftModel):
|
||||
block_size=int(self.block_size),
|
||||
rms_norm_eps=self.rms_norm_eps,
|
||||
config=parse_lilicorr_draft_config(draft_hf_config=config),
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("lilicorr", prefix),
|
||||
)
|
||||
# Every TP rank runs the replicated head, so its draft must match bit for bit.
|
||||
# Marlin's atomic-add reduce at the head's shapes does not.
|
||||
if get_parallel().tp_size > 1 and any(
|
||||
_runs_fp4_marlin(m) for m in self.lilicorr.modules()
|
||||
):
|
||||
raise ValueError("A LiLiCorr head on NVFP4 Marlin requires tp_size 1.")
|
||||
|
||||
def set_block_size(self, block_size: int) -> None:
|
||||
super().set_block_size(block_size)
|
||||
@@ -466,7 +568,7 @@ class LiLiCorrDraftModel(DFlashDraftModel):
|
||||
check_head_weight_coverage(self.lilicorr, seen)
|
||||
check_conv_weight_coverage(self, seen_conv)
|
||||
|
||||
parameter = next(self.lilicorr.parameters())
|
||||
parameter = self.lilicorr.slot_embedding
|
||||
self.lilicorr.materialize_inference_buffers(parameter.device, parameter.dtype)
|
||||
|
||||
|
||||
|
||||
@@ -7,10 +7,11 @@ import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.speculative.lilicorr import MAX_FUSED_CANDIDATE_TOPK
|
||||
from sglang.srt.distributed.communication_op import tensor_model_parallel_all_gather
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.models.dflash import candidate_topk
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.speculative.dflash_utils import _get_dflash_config
|
||||
from sglang.srt.speculative.dspark_components.dspark_draft import resolve_greedy_mask
|
||||
|
||||
@@ -193,7 +194,7 @@ def propose_lilicorr_block(
|
||||
)
|
||||
candidate_tokens = candidate_tokens.view(bs, slots, int(head.candidate_topk))
|
||||
|
||||
feat = int(head.context_proj.in_features)
|
||||
feat = int(head.context_proj.input_size)
|
||||
if anchor is None or int(anchor.shape[0]) != bs:
|
||||
# The batch resized since the anchor write; score invalid rather than misaligned.
|
||||
anchor_hidden = torch.zeros(
|
||||
@@ -258,7 +259,7 @@ class LiLiCorrDraftSampler:
|
||||
|
||||
# From the head, not from lm_head.weight: a quantized head's weight is packed,
|
||||
# so its dtype is not the dtype these buffers carry.
|
||||
anchor_row = next(head.parameters())
|
||||
anchor_row = head.slot_embedding
|
||||
device, dtype = anchor_row.device, anchor_row.dtype
|
||||
max_rows = self.max_bs * self.slots
|
||||
|
||||
@@ -267,7 +268,12 @@ class LiLiCorrDraftSampler:
|
||||
(self.max_bs, int(anchor_features)), dtype=dtype, device=device
|
||||
)
|
||||
self.anchor_valid = torch.zeros((self.max_bs,), dtype=torch.bool, device=device)
|
||||
self.token_table = head.build_token_table(embed_tokens)
|
||||
# Candidates carry global ids, so every rank holds the whole table.
|
||||
token_table = head.build_token_table(embed_tokens)
|
||||
if token_table is not None and getattr(embed_tokens, "tp_size", 1) > 1:
|
||||
token_table = tensor_model_parallel_all_gather(token_table, dim=0)
|
||||
assert token_table.shape[0] == embed_tokens.org_vocab_size_padded
|
||||
self.token_table = token_table
|
||||
|
||||
self.temperatures = torch.ones(
|
||||
(self.max_bs,), dtype=torch.float32, device=device
|
||||
@@ -386,16 +392,13 @@ def build_lilicorr_draft_sampler(
|
||||
)
|
||||
return None
|
||||
|
||||
tp_group = get_parallel().tp_group
|
||||
if int(tp_group.world_size) != 1:
|
||||
return eager("tp>1")
|
||||
batch_sizes = draft_graph_batch_sizes()
|
||||
if not batch_sizes:
|
||||
return eager("no draft graph batch sizes to size the static buffers from")
|
||||
max_bs = batch_sizes[-1]
|
||||
|
||||
reject_added_vocab(lm_head)
|
||||
parameter = next(head.parameters())
|
||||
parameter = head.slot_embedding
|
||||
head.materialize_inference_buffers(parameter.device, parameter.dtype)
|
||||
|
||||
sampler = LiLiCorrDraftSampler(
|
||||
@@ -404,7 +407,7 @@ def build_lilicorr_draft_sampler(
|
||||
lm_head=lm_head,
|
||||
block_size=int(block_size),
|
||||
max_bs=int(max_bs),
|
||||
anchor_features=int(draft_model.fc.out_features),
|
||||
anchor_features=int(draft_model.fc.output_size),
|
||||
sampling_enabled=sampling_enabled,
|
||||
)
|
||||
logger.info(
|
||||
|
||||
Reference in New Issue
Block a user