[Spec] Model-agnostic last-stage draft embedding under pipeline parallelism (#39643)

This commit is contained in:
YAMY
2026-09-28 10:22:40 -07:00
committed by GitHub
parent 6624999385
commit f209e67767
8 changed files with 491 additions and 190 deletions
@@ -10,7 +10,6 @@ from typing import Any, Dict, List, Optional
from sglang.srt.arg_groups.overrides import (
_hisparse_validation,
model_config_of,
resolved_view,
resolving_view,
run_post_process_pass,
@@ -25,21 +24,6 @@ from sglang.srt.utils.runai_utils import is_runai_obj_uri
logger = logging.getLogger(__name__)
_PP_EAGLE_SUPPORTED_ARCHITECTURES = frozenset(
{
"DeepseekV2ForCausalLM",
"DeepseekV3ForCausalLM",
"DeepseekV32ForCausalLM",
"GlmMoeDsaForCausalLM",
# Qwen3.5 (dense / MoE, text and multimodal); folded in from #39602.
"Qwen3_5ForCausalLM",
"Qwen3_5MoeForCausalLM",
"Qwen3_5ForConditionalGeneration",
"Qwen3_5MoeForConditionalGeneration",
"Qwen4ExpForConditionalGeneration",
}
)
def validate_response_store(server_args: Any) -> None:
cfg = resolving_view(server_args)
@@ -51,9 +35,7 @@ def validate_response_store(server_args: Any) -> None:
)
def check_pipeline_parallel_compat(
cfg: Any, *, model_architecture: Optional[str] = None
) -> None:
def check_pipeline_parallel_compat(cfg: Any) -> None:
"""Validate features used with pipeline parallelism."""
assert cfg.disable_overlap_schedule, (
"Pipeline parallelism is not compatible with overlap schedule"
@@ -98,11 +80,6 @@ def check_pipeline_parallel_compat(
"PP + speculative decoding (MTP) is only supported on prefill nodes "
"(disaggregation-mode=prefill)"
)
assert model_architecture in _PP_EAGLE_SUPPORTED_ARCHITECTURES, (
"PP + speculative decoding is only supported for DeepSeek/GLM/Qwen3.5 "
"models whose last pipeline stage supplies the EAGLE draft "
f"embedding; got architecture={model_architecture}"
)
assert cfg.min_free_slots_delay is None, (
"--min-free-slots-delay is not supported with pipeline "
"parallelism: allocatable slots per microbatch are bounded by "
@@ -135,10 +112,7 @@ def check_server_args(server_args: Any):
)
if cfg.pp_size > 1:
model_architecture = None
if cfg.speculative_algorithm is not None:
model_architecture = model_config_of(server_args).hf_config.architectures[0]
check_pipeline_parallel_compat(cfg, model_architecture=model_architecture)
check_pipeline_parallel_compat(cfg)
assert not (cfg.dp_size > 1 and cfg.nnodes != 1 and not cfg.enable_dp_attention), (
"multi-node data parallel is not supported unless dp attention!"
+12 -105
View File
@@ -101,6 +101,7 @@ from sglang.srt.speculative.eagle_worker_common import (
prepare_for_draft_extend,
run_eagle_verify,
)
from sglang.srt.speculative.pp_draft_embedding import resolve_draft_embed_and_head
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.speculative.spec_utils import (
draft_pp_context,
@@ -142,85 +143,6 @@ _is_xpu = is_xpu()
logger = logging.getLogger(__name__)
# Checkpoint spellings of the input embedding across the model families the
# PP+spec gate admits (GLM/DeepSeek NextN, Bailing MTP, Mistral-style drafts).
_EMBED_TENSOR_NAMES = (
"model.embed_tokens.weight",
"embed.weight",
"model.word_embeddings.weight",
"tok_embeddings.weight",
)
def _find_draft_input_embedding(model) -> "torch.nn.Module":
"""The draft's input embedding, found by type rather than attribute path.
Draft models hang it under different names (embed_tokens, word_embeddings,
embed, tok_embeddings), but it is always the one VocabParallelEmbedding
that is not the ParallelLMHead."""
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
found = [
(name, module)
for name, module in model.named_modules()
if isinstance(module, VocabParallelEmbedding)
and not isinstance(module, ParallelLMHead)
]
if len(found) != 1:
raise ValueError(
"PP+spec needs exactly one input embedding on the draft model, "
f"found {[name for name, _ in found]!r}"
)
return found[0][1]
def _load_checkpoint_tensor(
model_path: str, revision, tensor_names: tuple, load_config
) -> torch.Tensor:
"""Load one tensor from a checkpoint via the standard weight loader."""
from sglang.srt.configs.load_config import LoadFormat
from sglang.srt.model_loader.loader import DefaultModelLoader
from sglang.srt.model_loader.weight_utils import (
pt_weights_iterator,
safetensors_weights_iterator,
)
# Streaming and cache-transport formats have no weight files this helper
# could reopen; the dummy format is already skipped by the caller.
reopenable = (
LoadFormat.AUTO,
LoadFormat.SAFETENSORS,
LoadFormat.FASTSAFETENSORS,
LoadFormat.MISTRAL,
LoadFormat.PT,
LoadFormat.NPCACHE,
)
if load_config.load_format not in reopenable:
raise ValueError(
"PP+spec draft embedding loading cannot re-open weights under "
f"load format {load_config.load_format!r}; use a disk-backed "
"load format or disable SGLANG_ENABLE_PP_SPEC"
)
# The target's own load config keeps --download-dir, ignore patterns and
# the selected format, so hub ids resolve into the same cache the model
# was loaded from instead of a fresh default-location download.
_, weight_files, use_safetensors = DefaultModelLoader(load_config)._prepare_weights(
model_path, revision, fall_back_to_pt=True
)
iterator = (
safetensors_weights_iterator(weight_files)
if use_safetensors
else pt_weights_iterator(weight_files)
)
for name, tensor in iterator:
if name in tensor_names:
return tensor
raise ValueError(f"none of {tensor_names} found in checkpoint at {model_path}")
def _qsa_index_share_requested(hf_config) -> bool:
"""--json-model-override-args writes top-level hf_config attributes, while
checkpoint configs carry the flag on the nested text_config; read both."""
@@ -403,32 +325,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
def init_lm_head(self):
from sglang.srt.lora.layers import unwrap_lora_layer
if envs.SGLANG_ENABLE_PP_SPEC.get() and get_parallel().pp_size > 1:
# This branch skips the hot-token-map / EAGLE3 head wiring below.
assert self.hot_token_id is None and not (
self.speculative_algorithm.is_eagle3()
), "PP+spec does not support --speculative-token-map or EAGLE3 drafts yet"
# PP+spec: the target's embedding lives on the first PP stage
# (PPMissingLayer here on the last stage) and NextN/MTP layers
# carry no embedding of their own in the checkpoint, so the
# draft's embedding must be loaded from the checkpoint directly
# — otherwise it stays randomly initialized and accept_length
# collapses to ~1.
embed = _find_draft_input_embedding(self.draft_runner.model).weight
if get_model().load_format != "dummy":
target_runner = self.target_worker.model_runner
loaded_embed = _load_checkpoint_tensor(
model_path=target_runner.model_config.model_path,
revision=target_runner.model_config.revision,
tensor_names=_EMBED_TENSOR_NAMES,
load_config=target_runner.load_config,
)
embed.weight_loader(embed, loaded_embed)
head = self.target_worker.model_runner.model.lm_head.weight
self.draft_runner.model.set_embed_and_head(embed, head)
return
embed, head = self.target_worker.model_runner.model.get_embed_and_head()
embed, head = self._resolve_shared_embed_and_head()
target_lm_head = unwrap_lora_layer(
getattr(self.target_worker.model_runner.model, "lm_head", None)
)
@@ -470,6 +367,16 @@ class EagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner.model.set_embed_and_head(embed, head)
maybe_share_target_lm_head()
def _resolve_shared_embed_and_head(self):
target_runner = self.target_worker.model_runner
return resolve_draft_embed_and_head(
target_model=target_runner.model,
draft_model=self.draft_runner.model,
model_path=target_runner.model_config.model_path,
revision=target_runner.model_config.revision,
load_config=target_runner.load_config,
)
def init_attention_backend(self):
# Create multi-step attn backends and cuda graph runners
@@ -69,6 +69,7 @@ from sglang.srt.speculative.frozen_kv_mtp_utils import (
set_frozen_kv_positions,
target_kv_pool_view,
)
from sglang.srt.speculative.pp_draft_embedding import resolve_draft_embed_and_head
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.speculative.spec_utils import (
draft_pp_context,
@@ -148,8 +149,15 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
context_length=self.target_worker.model_runner.model_config.context_len,
)
embed, head = self.target_worker.model_runner.model.get_embed_and_head()
if hasattr(self.draft_model_runner.model, "set_embed_and_head"):
target_runner = self.target_worker.model_runner
embed, head = resolve_draft_embed_and_head(
target_model=target_runner.model,
draft_model=self.draft_model_runner.model,
model_path=target_runner.model_config.model_path,
revision=target_runner.model_config.revision,
load_config=target_runner.load_config,
)
self.draft_model_runner.model.set_embed_and_head(embed, head)
else:
logger.debug(
@@ -79,6 +79,7 @@ from sglang.srt.speculative.multi_layer_eagle_utils import (
rotate_input_ids,
stash_append_boundary_state_triton,
)
from sglang.srt.speculative.pp_draft_embedding import resolve_draft_embed_and_head
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.speculative.spec_utils import (
draft_pp_context,
@@ -367,9 +368,16 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
)
def init_lm_head(self):
embed, head = self.target_worker.model_runner.model.get_embed_and_head()
target_runner = self.target_worker.model_runner
# Share the embedding and lm_head
for i in range(self.speculative_num_steps):
embed, head = resolve_draft_embed_and_head(
target_model=target_runner.model,
draft_model=self.draft_runner_list[i].model,
model_path=target_runner.model_config.model_path,
revision=target_runner.model_config.revision,
load_config=target_runner.load_config,
)
self.draft_runner_list[i].model.set_embed_and_head(embed, head)
def init_attention_backend(self):
@@ -0,0 +1,241 @@
"""Model-agnostic draft embedding for speculative decoding under pipeline parallelism.
The draft runs only on the last stage while the target embedding lives only on the
first stage, so the draft cannot share ``target.get_embed_and_head()``. This module
reads the target embed/head PP-safely and loads the checkpoint's input embedding
into the draft's own embedding parameter, touching only the shard that holds it.
"""
from __future__ import annotations
import json
import logging
import os
import re
from typing import Iterable, List, Optional, Tuple
import torch
from torch import nn
from sglang.srt.configs.load_config import LoadConfig, LoadFormat
from sglang.srt.layers.utils.common import PPMissingLayer
from sglang.srt.model_loader.weight_utils import default_weight_loader
logger = logging.getLogger(__name__)
SAFETENSORS_INDEX_NAME = "model.safetensors.index.json"
# Module attribute names under which model families hang the input embedding.
_EMBED_ATTR_NAMES: Tuple[str, ...] = (
"embed_tokens",
"word_embeddings",
"tok_embeddings",
"embed",
)
# Checkpoint spellings of the input embedding, most common first.
EMBED_KEY_CANDIDATES: Tuple[str, ...] = (
"model.embed_tokens.weight",
"model.language_model.embed_tokens.weight",
"language_model.model.embed_tokens.weight",
"model.word_embeddings.weight",
"tok_embeddings.weight",
"embed_tokens.weight",
"embed.weight",
)
_EMBED_KEY_SUFFIXES: Tuple[str, ...] = tuple(f"{n}.weight" for n in _EMBED_ATTR_NAMES)
# MTP / NextN layers carry their own embedding under ``layers.<n>.``; never pick it.
_LAYER_KEY_RE = re.compile(r"(^|\.)layers\.\d+\.")
_DRAFT_SUBMODULE_MARKERS: Tuple[str, ...] = ("mtp", "nextn", "eagle", "draft")
# Streaming and cache-transport formats have no weight files to re-open.
_REOPENABLE_LOAD_FORMATS = (
LoadFormat.AUTO,
LoadFormat.SAFETENSORS,
LoadFormat.FASTSAFETENSORS,
LoadFormat.MISTRAL,
LoadFormat.PT,
LoadFormat.NPCACHE,
)
def _target_input_embedding_is_missing(target_model: nn.Module) -> bool:
"""True when the target's input embedding on this stage is a ``PPMissingLayer``."""
for name, module in target_model.named_modules():
if name.rsplit(".", 1)[-1] in _EMBED_ATTR_NAMES and isinstance(
module, PPMissingLayer
):
return True
return False
def _weight_or_none(module: Optional[nn.Module]) -> Optional[torch.Tensor]:
if module is None or isinstance(module, PPMissingLayer):
return None
from sglang.srt.lora.layers import unwrap_lora_layer
return unwrap_lora_layer(module).weight
def resolve_target_embed_and_head(
target_model: nn.Module,
) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
"""PP-safe getter: a PPMissingLayer embedding maps to ``embed=None``; any
other AttributeError propagates."""
try:
return target_model.get_embed_and_head()
except AttributeError:
if not _target_input_embedding_is_missing(target_model):
raise
return None, _weight_or_none(target_model.lm_head)
def find_draft_embedding_param(
draft_model: nn.Module,
) -> Optional[Tuple[str, nn.Parameter]]:
"""The draft's own input embedding: its single ``VocabParallelEmbedding``."""
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
found = [
(f"{name}.weight", module.weight)
for name, module in draft_model.named_modules()
if isinstance(module, VocabParallelEmbedding)
and not isinstance(module, ParallelLMHead)
]
return found[0] if len(found) == 1 else None
def _is_input_embedding_key(key: str) -> bool:
if not key.endswith(_EMBED_KEY_SUFFIXES) or _LAYER_KEY_RE.search(key):
return False
lowered = key.lower()
return not any(marker in lowered for marker in _DRAFT_SUBMODULE_MARKERS)
def _pick_embedding_key(keys: Iterable[str]) -> Optional[str]:
keys = set(keys)
for candidate in EMBED_KEY_CANDIDATES:
if candidate in keys:
return candidate
fallback = sorted((k for k in keys if _is_input_embedding_key(k)), key=len)
return fallback[0] if fallback else None
def _safetensors_keys(path: str) -> List[str]:
import safetensors
with safetensors.safe_open(path, framework="pt", device="cpu") as f:
return list(f.keys())
def _read_safetensors_tensor(path: str, key: str) -> torch.Tensor:
import safetensors
with safetensors.safe_open(path, framework="pt", device="cpu") as f:
return f.get_tensor(key)
def prepare_checkpoint_files(
model_path: str, *, revision: Optional[str], load_config: LoadConfig
) -> Tuple[str, List[str], bool]:
"""``(folder, weight_files, use_safetensors)`` via the standard model loader,
so ModelScope resolution, ``--download-dir`` and ``load_format`` are honored."""
from sglang.srt.model_loader.loader import DefaultModelLoader
if load_config.load_format not in _REOPENABLE_LOAD_FORMATS:
raise ValueError(
"Pipeline-parallel speculative decoding needs to re-open the target "
f"checkpoint for the draft embedding, which load format "
f"{load_config.load_format!r} does not allow; use a disk-backed format."
)
return DefaultModelLoader(load_config)._prepare_weights(
model_path, revision, fall_back_to_pt=True
)
def load_embedding_tensor(
folder: str, weight_files: List[str], *, use_safetensors: bool
) -> Tuple[str, torch.Tensor]:
"""``(key, tensor)`` of the checkpoint's input embedding; with an index only
the owning shard is read, otherwise the key is picked over all shard headers."""
if use_safetensors:
index_file = os.path.join(folder, SAFETENSORS_INDEX_NAME)
if os.path.exists(index_file):
with open(index_file) as f:
weight_map = json.load(f).get("weight_map", {}) or {}
key = _pick_embedding_key(weight_map.keys())
if key is not None:
shard = os.path.join(folder, weight_map[key])
return key, _read_safetensors_tensor(shard, key)
keys_by_file = {path: _safetensors_keys(path) for path in weight_files}
key = _pick_embedding_key(k for keys in keys_by_file.values() for k in keys)
if key is not None:
shard = next(p for p, keys in keys_by_file.items() if key in keys)
return key, _read_safetensors_tensor(shard, key)
else:
from sglang.srt.model_loader.weight_utils import pt_weights_iterator
for name, tensor in pt_weights_iterator(weight_files):
if name in EMBED_KEY_CANDIDATES or _is_input_embedding_key(name):
return name, tensor
raise ValueError(
f"No input embedding found in checkpoint under {folder}; looked for "
f"{EMBED_KEY_CANDIDATES} or '*.<{'|'.join(_EMBED_ATTR_NAMES)}>.weight'."
)
def load_draft_embedding_from_checkpoint(
draft_model: nn.Module,
model_path: str,
*,
revision: Optional[str],
load_config: LoadConfig,
) -> nn.Parameter:
"""Load the checkpoint's input embedding into the draft's own parameter and
return it; the parameter's ``weight_loader`` applies the TP shard."""
found = find_draft_embedding_param(draft_model)
if found is None:
raise ValueError(
f"Draft model {draft_model.__class__.__name__} has no single "
"VocabParallelEmbedding, and the target cannot share its embedding from "
"this pipeline stage (https://github.com/sgl-project/sglang/issues/39634)."
)
param_name, param = found
if load_config.load_format == LoadFormat.DUMMY:
return param
folder, weight_files, use_safetensors = prepare_checkpoint_files(
model_path, revision=revision, load_config=load_config
)
key, loaded = load_embedding_tensor(
folder, weight_files, use_safetensors=use_safetensors
)
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded)
logger.info(
"Loaded draft embedding %s from checkpoint key %s %s for pipeline-parallel "
"speculative decoding.",
param_name,
key,
tuple(loaded.shape),
)
return param
def resolve_draft_embed_and_head(
*,
target_model: nn.Module,
draft_model: nn.Module,
model_path: str,
revision: Optional[str],
load_config: LoadConfig,
) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
"""Embed/head to bind into a draft; a stage without the target embedding
loads the draft's own from the checkpoint."""
embed, head = resolve_target_embed_and_head(target_model)
if embed is None:
embed = load_draft_embedding_from_checkpoint(
draft_model, model_path, revision=revision, load_config=load_config
)
return embed, head