242 lines
8.7 KiB
Python
242 lines
8.7 KiB
Python
"""Model-agnostic draft embedding for speculative decoding under pipeline parallelism.
|
|
|
|
The draft runs only on the last stage while the target embedding lives only on the
|
|
first stage, so the draft cannot share ``target.get_embed_and_head()``. This module
|
|
reads the target embed/head PP-safely and loads the checkpoint's input embedding
|
|
into the draft's own embedding parameter, touching only the shard that holds it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
from typing import Iterable, List, Optional, Tuple
|
|
|
|
import torch
|
|
from torch import nn
|
|
|
|
from sglang.srt.configs.load_config import LoadConfig, LoadFormat
|
|
from sglang.srt.layers.utils.common import PPMissingLayer
|
|
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
SAFETENSORS_INDEX_NAME = "model.safetensors.index.json"
|
|
|
|
# Module attribute names under which model families hang the input embedding.
|
|
_EMBED_ATTR_NAMES: Tuple[str, ...] = (
|
|
"embed_tokens",
|
|
"word_embeddings",
|
|
"tok_embeddings",
|
|
"embed",
|
|
)
|
|
# Checkpoint spellings of the input embedding, most common first.
|
|
EMBED_KEY_CANDIDATES: Tuple[str, ...] = (
|
|
"model.embed_tokens.weight",
|
|
"model.language_model.embed_tokens.weight",
|
|
"language_model.model.embed_tokens.weight",
|
|
"model.word_embeddings.weight",
|
|
"tok_embeddings.weight",
|
|
"embed_tokens.weight",
|
|
"embed.weight",
|
|
)
|
|
_EMBED_KEY_SUFFIXES: Tuple[str, ...] = tuple(f"{n}.weight" for n in _EMBED_ATTR_NAMES)
|
|
# MTP / NextN layers carry their own embedding under ``layers.<n>.``; never pick it.
|
|
_LAYER_KEY_RE = re.compile(r"(^|\.)layers\.\d+\.")
|
|
_DRAFT_SUBMODULE_MARKERS: Tuple[str, ...] = ("mtp", "nextn", "eagle", "draft")
|
|
# Streaming and cache-transport formats have no weight files to re-open.
|
|
_REOPENABLE_LOAD_FORMATS = (
|
|
LoadFormat.AUTO,
|
|
LoadFormat.SAFETENSORS,
|
|
LoadFormat.FASTSAFETENSORS,
|
|
LoadFormat.MISTRAL,
|
|
LoadFormat.PT,
|
|
LoadFormat.NPCACHE,
|
|
)
|
|
|
|
|
|
def _target_input_embedding_is_missing(target_model: nn.Module) -> bool:
|
|
"""True when the target's input embedding on this stage is a ``PPMissingLayer``."""
|
|
for name, module in target_model.named_modules():
|
|
if name.rsplit(".", 1)[-1] in _EMBED_ATTR_NAMES and isinstance(
|
|
module, PPMissingLayer
|
|
):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _weight_or_none(module: Optional[nn.Module]) -> Optional[torch.Tensor]:
|
|
if module is None or isinstance(module, PPMissingLayer):
|
|
return None
|
|
from sglang.srt.lora.layers import unwrap_lora_layer
|
|
|
|
return unwrap_lora_layer(module).weight
|
|
|
|
|
|
def resolve_target_embed_and_head(
|
|
target_model: nn.Module,
|
|
) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
|
|
"""PP-safe getter: a PPMissingLayer embedding maps to ``embed=None``; any
|
|
other AttributeError propagates."""
|
|
try:
|
|
return target_model.get_embed_and_head()
|
|
except AttributeError:
|
|
if not _target_input_embedding_is_missing(target_model):
|
|
raise
|
|
return None, _weight_or_none(target_model.lm_head)
|
|
|
|
|
|
def find_draft_embedding_param(
|
|
draft_model: nn.Module,
|
|
) -> Optional[Tuple[str, nn.Parameter]]:
|
|
"""The draft's own input embedding: its single ``VocabParallelEmbedding``."""
|
|
from sglang.srt.layers.vocab_parallel_embedding import (
|
|
ParallelLMHead,
|
|
VocabParallelEmbedding,
|
|
)
|
|
|
|
found = [
|
|
(f"{name}.weight", module.weight)
|
|
for name, module in draft_model.named_modules()
|
|
if isinstance(module, VocabParallelEmbedding)
|
|
and not isinstance(module, ParallelLMHead)
|
|
]
|
|
return found[0] if len(found) == 1 else None
|
|
|
|
|
|
def _is_input_embedding_key(key: str) -> bool:
|
|
if not key.endswith(_EMBED_KEY_SUFFIXES) or _LAYER_KEY_RE.search(key):
|
|
return False
|
|
lowered = key.lower()
|
|
return not any(marker in lowered for marker in _DRAFT_SUBMODULE_MARKERS)
|
|
|
|
|
|
def _pick_embedding_key(keys: Iterable[str]) -> Optional[str]:
|
|
keys = set(keys)
|
|
for candidate in EMBED_KEY_CANDIDATES:
|
|
if candidate in keys:
|
|
return candidate
|
|
fallback = sorted((k for k in keys if _is_input_embedding_key(k)), key=len)
|
|
return fallback[0] if fallback else None
|
|
|
|
|
|
def _safetensors_keys(path: str) -> List[str]:
|
|
import safetensors
|
|
|
|
with safetensors.safe_open(path, framework="pt", device="cpu") as f:
|
|
return list(f.keys())
|
|
|
|
|
|
def _read_safetensors_tensor(path: str, key: str) -> torch.Tensor:
|
|
import safetensors
|
|
|
|
with safetensors.safe_open(path, framework="pt", device="cpu") as f:
|
|
return f.get_tensor(key)
|
|
|
|
|
|
def prepare_checkpoint_files(
|
|
model_path: str, *, revision: Optional[str], load_config: LoadConfig
|
|
) -> Tuple[str, List[str], bool]:
|
|
"""``(folder, weight_files, use_safetensors)`` via the standard model loader,
|
|
so ModelScope resolution, ``--download-dir`` and ``load_format`` are honored."""
|
|
from sglang.srt.model_loader.loader import DefaultModelLoader
|
|
|
|
if load_config.load_format not in _REOPENABLE_LOAD_FORMATS:
|
|
raise ValueError(
|
|
"Pipeline-parallel speculative decoding needs to re-open the target "
|
|
f"checkpoint for the draft embedding, which load format "
|
|
f"{load_config.load_format!r} does not allow; use a disk-backed format."
|
|
)
|
|
return DefaultModelLoader(load_config)._prepare_weights(
|
|
model_path, revision, fall_back_to_pt=True
|
|
)
|
|
|
|
|
|
def load_embedding_tensor(
|
|
folder: str, weight_files: List[str], *, use_safetensors: bool
|
|
) -> Tuple[str, torch.Tensor]:
|
|
"""``(key, tensor)`` of the checkpoint's input embedding; with an index only
|
|
the owning shard is read, otherwise the key is picked over all shard headers."""
|
|
if use_safetensors:
|
|
index_file = os.path.join(folder, SAFETENSORS_INDEX_NAME)
|
|
if os.path.exists(index_file):
|
|
with open(index_file) as f:
|
|
weight_map = json.load(f).get("weight_map", {}) or {}
|
|
key = _pick_embedding_key(weight_map.keys())
|
|
if key is not None:
|
|
shard = os.path.join(folder, weight_map[key])
|
|
return key, _read_safetensors_tensor(shard, key)
|
|
keys_by_file = {path: _safetensors_keys(path) for path in weight_files}
|
|
key = _pick_embedding_key(k for keys in keys_by_file.values() for k in keys)
|
|
if key is not None:
|
|
shard = next(p for p, keys in keys_by_file.items() if key in keys)
|
|
return key, _read_safetensors_tensor(shard, key)
|
|
else:
|
|
from sglang.srt.model_loader.weight_utils import pt_weights_iterator
|
|
|
|
for name, tensor in pt_weights_iterator(weight_files):
|
|
if name in EMBED_KEY_CANDIDATES or _is_input_embedding_key(name):
|
|
return name, tensor
|
|
raise ValueError(
|
|
f"No input embedding found in checkpoint under {folder}; looked for "
|
|
f"{EMBED_KEY_CANDIDATES} or '*.<{'|'.join(_EMBED_ATTR_NAMES)}>.weight'."
|
|
)
|
|
|
|
|
|
def load_draft_embedding_from_checkpoint(
|
|
draft_model: nn.Module,
|
|
model_path: str,
|
|
*,
|
|
revision: Optional[str],
|
|
load_config: LoadConfig,
|
|
) -> nn.Parameter:
|
|
"""Load the checkpoint's input embedding into the draft's own parameter and
|
|
return it; the parameter's ``weight_loader`` applies the TP shard."""
|
|
found = find_draft_embedding_param(draft_model)
|
|
if found is None:
|
|
raise ValueError(
|
|
f"Draft model {draft_model.__class__.__name__} has no single "
|
|
"VocabParallelEmbedding, and the target cannot share its embedding from "
|
|
"this pipeline stage (https://github.com/sgl-project/sglang/issues/39634)."
|
|
)
|
|
param_name, param = found
|
|
if load_config.load_format == LoadFormat.DUMMY:
|
|
return param
|
|
|
|
folder, weight_files, use_safetensors = prepare_checkpoint_files(
|
|
model_path, revision=revision, load_config=load_config
|
|
)
|
|
key, loaded = load_embedding_tensor(
|
|
folder, weight_files, use_safetensors=use_safetensors
|
|
)
|
|
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
|
weight_loader(param, loaded)
|
|
logger.info(
|
|
"Loaded draft embedding %s from checkpoint key %s %s for pipeline-parallel "
|
|
"speculative decoding.",
|
|
param_name,
|
|
key,
|
|
tuple(loaded.shape),
|
|
)
|
|
return param
|
|
|
|
|
|
def resolve_draft_embed_and_head(
|
|
*,
|
|
target_model: nn.Module,
|
|
draft_model: nn.Module,
|
|
model_path: str,
|
|
revision: Optional[str],
|
|
load_config: LoadConfig,
|
|
) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
|
|
"""Embed/head to bind into a draft; a stage without the target embedding
|
|
loads the draft's own from the checkpoint."""
|
|
embed, head = resolve_target_embed_and_head(target_model)
|
|
if embed is None:
|
|
embed = load_draft_embedding_from_checkpoint(
|
|
draft_model, model_path, revision=revision, load_config=load_config
|
|
)
|
|
return embed, head
|