[Refactor] Use destination layouts for model checkpoint mappings (#42590)

This commit is contained in:
Cheng Wan
2026-10-07 16:24:22 -07:00
committed by GitHub
parent 5e12831cec
commit 7760d5264a
25 changed files with 1725 additions and 72 deletions
@@ -27,6 +27,7 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary import get_attn_tp_context
from sglang.srt.layers.linear import _resolve_linear_group
from sglang.srt.layers.parameter import BasevLLMParameter
from sglang.srt.layers.quantization.base_config import (
QuantizationConfig,
@@ -248,6 +249,11 @@ class VocabParallelEmbedding(torch.nn.Module):
self.enable_tp = enable_tp
self.use_attn_tp_group = use_attn_tp_group
self.tp_group = (
_resolve_linear_group("attn_tp" if use_attn_tp_group else "tp")
if enable_tp
else None
)
if self.enable_tp:
if use_attn_tp_group:
tp_rank = get_parallel().attn_tp_rank
+3 -2
View File
@@ -39,6 +39,7 @@ from sglang.srt.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
from sglang.srt.layers.pooler import Pooler, PoolingType
@@ -311,6 +312,7 @@ class ApertusModel(nn.Module):
self.vocab_size = config.vocab_size
self.org_vocab_size = config.vocab_size
self.pp_group = get_parallel().pp_group
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
if self.pp_group.is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
@@ -381,8 +383,7 @@ class ApertusModel(nn.Module):
# factors (or else raise an exception). Thus, handled exceptions should
# make sure to leave KV cache scale factors in a known good (dummy) state
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
tp_size = get_parallel().tp_size
tp_rank = get_parallel().tp_rank
tp_rank, tp_size = self._kv_cache_parallel_layout
for layer_idx, scaling_factor in kv_cache_scales_loader(
quantization_param_path,
tp_rank,
+3 -2
View File
@@ -33,6 +33,7 @@ from sglang.srt.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
from sglang.srt.layers.pooler import Pooler, PoolingType
@@ -285,6 +286,7 @@ class ArceeModel(nn.Module):
self.padding_idx = config.pad_token_id
self.vocab_size = config.vocab_size
self.pp_group = get_parallel().pp_group
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
if self.pp_group.is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
@@ -352,8 +354,7 @@ class ArceeModel(nn.Module):
return hidden_states, aux_hidden_states
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
tp_size = get_parallel().tp_size
tp_rank = get_parallel().tp_rank
tp_rank, tp_size = self._kv_cache_parallel_layout
for layer_idx, scaling_factor in kv_cache_scales_loader(
quantization_param_path,
tp_rank,
@@ -92,6 +92,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.managers.mm_utils import (
MultiModalityDataPaddingPatternTokenPairs,
general_mm_embed_routine,
@@ -1937,12 +1938,17 @@ class Dots3LanguageModelForCausalLM(nn.Module):
"Dots3 requires q_lora_rank to enable fused_qkv_a_g_proj_with_mqa loading."
)
cached_a_proj = {} if fuse_qkv_a_g_proj else None
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
def shard_g_proj_for_attention_tp(
weight: torch.Tensor, cat_dim: int, is_scale: bool
weight: torch.Tensor,
cat_dim: int,
is_scale: bool,
*,
q_b_proj: ColumnParallelLinear,
):
group = unwrap_lora_layer(q_b_proj).tp_group
attn_tp_rank = group.rank_in_group if group is not None else 0
attn_tp_size = group.world_size if group is not None else 1
assert weight.ndim > cat_dim, (
f"weight.ndim={weight.ndim}, cat_dim={cat_dim}"
)
@@ -1991,7 +1997,10 @@ class Dots3LanguageModelForCausalLM(nn.Module):
kv_a_proj_weight = cached_a_proj[kv_a_proj_name]
g_proj_weight = cached_a_proj[g_proj_name]
g_proj_shard = shard_g_proj_for_attention_tp(
g_proj_weight, cat_dim, is_scale
g_proj_weight,
cat_dim,
is_scale,
q_b_proj=self.get_submodule(param_name.rsplit(".", 2)[0]).q_b_proj,
)
scale_block_n = _get_scale_block_n(self.quant_config)
kv_a_proj_weight_aligned = (
+27 -13
View File
@@ -28,19 +28,24 @@ from transformers import (
PreTrainedModel,
)
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.layers.activation import GeluAndMul
from sglang.srt.layers.layernorm import Gemma3RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
_resolve_linear_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.pooler import EmbeddingPoolerOutput, Pooler, PoolingType
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import AttentionType, RadixAttention
from sglang.srt.layers.rotary_embedding import apply_rotary_pos_emb, get_rope
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_loader.weight_utils import (
default_weight_loader,
@@ -735,6 +740,7 @@ class Gemma3ForCausalLM(PreTrainedModel):
super().__init__(config=config)
self.config = config
self.quant_config = quant_config
self._shared_vocab_tp_group = _resolve_linear_group("tp")
self.model = Gemma3TextModel(
config, quant_config, prefix=add_prefix("model", prefix)
)
@@ -919,28 +925,35 @@ class Gemma3ForCausalLM(PreTrainedModel):
# of the (i-1)th layer as aux hidden state
self.model.layers_to_capture = [val + 1 for val in layer_ids]
def _shard_weight(self, weight: torch.Tensor) -> torch.Tensor:
"""Shard a full embedding/lm_head weight along vocab dim for the current TP rank.
Gemma3 uses nn.Embedding (unsharded) but the Eagle3 draft model uses
VocabParallelEmbedding (sharded). This method extracts the correct
shard so the weights can be shared.
"""
tp_size = get_parallel().tp_size
def _shard_weight(
self, weight: torch.Tensor, *, draft_embedding=None
) -> torch.Tensor:
group = self._shared_vocab_tp_group
if draft_embedding is not None:
group = (
draft_embedding.tp_group
if isinstance(draft_embedding, VocabParallelEmbedding)
else None
)
tp_rank, tp_size = get_group_rank_size(group)
if tp_size <= 1:
return weight
tp_rank = get_parallel().tp_rank
shard_size = (weight.shape[0] + tp_size - 1) // tp_size
return weight[tp_rank * shard_size : (tp_rank + 1) * shard_size]
def get_embed(self):
return self._shard_weight(self.model.embed_tokens.weight)
def get_embed_and_head(self):
embed = self._shard_weight(self.model.embed_tokens.weight)
head = self._shard_weight(self.lm_head.weight)
def get_embed_and_head(self, *, draft_embedding=None):
embed = self._shard_weight(
self.model.embed_tokens.weight, draft_embedding=draft_embedding
)
head = self._shard_weight(self.lm_head.weight, draft_embedding=draft_embedding)
return embed, head
def get_embed_and_head_for_draft(self, draft_embedding):
return self.get_embed_and_head(draft_embedding=draft_embedding)
class EmbeddingGemmaModel(Gemma3ForCausalLM):
"""EmbeddingGemma's Gemma3 encoder with normalized mean pooling."""
@@ -957,6 +970,7 @@ class EmbeddingGemmaModel(Gemma3ForCausalLM):
PreTrainedModel.__init__(self, config=config)
self.config = config
self.quant_config = quant_config
self._shared_vocab_tp_group = _resolve_linear_group("tp")
self.model = Gemma3TextModel(
config, quant_config, prefix=add_prefix("model", prefix)
)
+3
View File
@@ -494,6 +494,9 @@ class Gemma3ForConditionalGeneration(PreTrainedModel):
# For EAGLE3, head might not be needed
return embed, None
def get_embed_and_head_for_draft(self, draft_embedding):
return self.language_model.get_embed_and_head_for_draft(draft_embedding)
def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None):
if hasattr(self.language_model, "set_eagle3_layers_to_capture"):
self.language_model.set_eagle3_layers_to_capture(layer_ids)
+26 -13
View File
@@ -33,6 +33,7 @@ from sglang.kernels.ops.moe.gemma4_routing import (
gemma4_fused_routing,
gemma_routing_post_topk,
)
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.layer_boundary import (
SumGroup,
@@ -48,6 +49,7 @@ from sglang.srt.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
_resolve_linear_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
@@ -57,7 +59,10 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_loader.weight_utils import (
default_weight_loader,
@@ -1131,6 +1136,7 @@ class Gemma4ForCausalLM(PreTrainedModel):
self.pp_group = get_parallel().pp_group
self.config = config
self.quant_config = quant_config
self._shared_vocab_tp_group = _resolve_linear_group("tp")
self.model = Gemma4TextModel(
config=config, quant_config=quant_config, prefix=add_prefix("model", prefix)
@@ -1425,24 +1431,26 @@ class Gemma4ForCausalLM(PreTrainedModel):
logger.log(level, "%s: %s", msg, names)
return loaded_params
def _shard_weight(self, weight: torch.Tensor) -> torch.Tensor:
"""Shard a full embedding/lm_head weight along vocab dim for the current TP rank.
Gemma4 uses nn.Embedding (unsharded) but the Eagle3 draft model uses
VocabParallelEmbedding (sharded). This method extracts the correct
shard so the weights can be shared.
"""
tp_size = get_parallel().tp_size
def _shard_weight(
self, weight: torch.Tensor, *, draft_embedding=None
) -> torch.Tensor:
group = self._shared_vocab_tp_group
if draft_embedding is not None:
group = (
draft_embedding.tp_group
if isinstance(draft_embedding, VocabParallelEmbedding)
else None
)
tp_rank, tp_size = get_group_rank_size(group)
if tp_size <= 1:
return weight
tp_rank = get_parallel().tp_rank
shard_size = (weight.shape[0] + tp_size - 1) // tp_size
return weight[tp_rank * shard_size : (tp_rank + 1) * shard_size]
def get_embed(self):
return self._shard_weight(self.model.embed_tokens.weight)
def get_embed_and_head(self):
def get_embed_and_head(self, *, draft_embedding=None):
if self.pp_group.world_size > 1:
# Under PP, embed_tokens lives on the first rank and lm_head on
# the last; neither rank holds both tensors, so we can't return
@@ -1456,10 +1464,15 @@ class Gemma4ForCausalLM(PreTrainedModel):
"PP rank and lm_head on the last; use --pp-size 1 if you "
"need this API."
)
embed = self._shard_weight(self.model.embed_tokens.weight)
head = self._shard_weight(self.lm_head.weight)
embed = self._shard_weight(
self.model.embed_tokens.weight, draft_embedding=draft_embedding
)
head = self._shard_weight(self.lm_head.weight, draft_embedding=draft_embedding)
return embed, head
def get_embed_and_head_for_draft(self, draft_embedding):
return self.get_embed_and_head(draft_embedding=draft_embedding)
def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None):
if layer_ids is None:
self.capture_aux_hidden_states = True
+3 -2
View File
@@ -38,6 +38,7 @@ from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.pooler import Pooler, PoolingType
@@ -318,6 +319,7 @@ class Glm4Model(nn.Module):
self.padding_idx = config.pad_token_id
self.vocab_size = config.vocab_size
self.pp_group = get_parallel().pp_group
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
if self.pp_group.is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
@@ -399,8 +401,7 @@ class Glm4Model(nn.Module):
# factors (or else raise an exception). Thus, handled exceptions should
# make sure to leave KV cache scale factors in a known good (dummy) state
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
tp_size = get_parallel().tp_size
tp_rank = get_parallel().tp_rank
tp_rank, tp_size = self._kv_cache_parallel_layout
for layer_idx, scaling_factor in kv_cache_scales_loader(
quantization_param_path,
tp_rank,
+7 -2
View File
@@ -27,6 +27,7 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
@@ -62,6 +63,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
get_tc_piecewise_forward_context,
@@ -1274,8 +1276,11 @@ class GptOssForCausalLM(nn.Module):
if name in params_dict.keys():
param = params_dict[name]
if "sinks" in name:
start = get_parallel().attn_tp_rank * param.numel()
tp_size = get_parallel().tp_size
projection = unwrap_lora_layer(
self.get_submodule(name.rsplit(".", 1)[0]).qkv_proj
)
tp_rank, tp_size = get_group_rank_size(projection.tp_group)
start = tp_rank * param.numel()
full_shard_size = param.numel() * tp_size
# This handles TP padding: if the checkpoint dim is not divisible by tp_size,
# the last TP shard extends beyond `loaded_weight`, pad with zeros before slicing.
+3 -2
View File
@@ -32,6 +32,7 @@ from sglang.srt.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
@@ -573,6 +574,7 @@ class HunYuanMoEV1ForCausalLM(nn.Module):
super().__init__()
self.config = config
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
self.model = HunYuanModel(config, quant_config, prefix="model")
self.unpadded_vocab_size = config.vocab_size
@@ -769,8 +771,7 @@ class HunYuanMoEV1ForCausalLM(nn.Module):
# factors (or else raise an exception). Thus, handled exceptions should
# make sure to leave KV cache scale factors in a known good (dummy) state
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
tp_size = get_parallel().tp_size
tp_rank = get_parallel().tp_rank
tp_rank, tp_size = self._kv_cache_parallel_layout
for layer_idx, scaling_factor in kv_cache_scales_loader(
quantization_param_path,
tp_rank,
+4 -3
View File
@@ -5,9 +5,11 @@ from typing import Any, Dict, Iterable, Optional, Tuple
import torch
from transformers import PretrainedConfig
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.qwen3_moe import Qwen3MoeAttention, Qwen3MoeDecoderLayer
@@ -15,7 +17,6 @@ from sglang.srt.models.qwen3_vl_moe import (
Qwen3MoeLLMModel,
Qwen3VLMoeForConditionalGeneration,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix
logger = logging.getLogger(__name__)
@@ -211,8 +212,8 @@ class InternS1ProForConditionalGeneration(Qwen3VLMoeForConditionalGeneration):
def _load_fope_weights(self, name: str, loaded_weight: torch.Tensor, params_dict):
"""load fope weights"""
attn_tp_size = get_parallel().attn_tp_size
attn_tp_rank = get_parallel().attn_tp_rank
qkv_proj = unwrap_lora_layer(self.model.layers[0].self_attn.qkv_proj)
attn_tp_rank, attn_tp_size = get_group_rank_size(qkv_proj.tp_group)
num_key_value_heads = loaded_weight.size(0)
# replicate head if necessary
+3 -2
View File
@@ -43,6 +43,7 @@ from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
from sglang.srt.layers.pooler import Pooler, PoolingType
@@ -406,6 +407,7 @@ class LlamaModel(nn.Module):
self.padding_idx = config.pad_token_id
self.vocab_size = config.vocab_size
self.pp_group = get_parallel().pp_group
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
if self.pp_group.is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
@@ -485,8 +487,7 @@ class LlamaModel(nn.Module):
# factors (or else raise an exception). Thus, handled exceptions should
# make sure to leave KV cache scale factors in a known good (dummy) state
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
tp_size = get_parallel().tp_size
tp_rank = get_parallel().tp_rank
tp_rank, tp_size = self._kv_cache_parallel_layout
for layer_idx, scaling_factor in kv_cache_scales_loader(
quantization_param_path,
tp_rank,
+23 -8
View File
@@ -24,6 +24,7 @@ from torch import nn
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -48,6 +49,7 @@ from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
@@ -65,6 +67,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.managers.mm_utils import (
MultiModalityDataPaddingPatternMultimodalTokens,
general_mm_embed_routine,
@@ -101,9 +104,11 @@ def load_mimo_v2_qkv_proj_weight(
loaded_weight,
expected_fused_tp_size: Optional[int] = None,
deferred_scale_inv: Optional[Dict[str, torch.Tensor]] = None,
*,
qkv_proj: QKVParallelLinear,
):
tp_size = get_parallel().attn_tp_size
tp_rank = get_parallel().attn_tp_rank
qkv_proj = unwrap_lora_layer(qkv_proj)
tp_rank, tp_size = get_group_rank_size(qkv_proj.tp_group)
ckpt_tp = expected_fused_tp_size if expected_fused_tp_size is not None else tp_size
if ckpt_tp == tp_size and loaded_weight.shape == param.shape:
@@ -194,11 +199,10 @@ def _resolve_deferred_qkv_scale_inv(
expected_fused_tp_size: int,
block_size: int = 128,
config=None,
*,
model: nn.Module,
):
tp_size = get_parallel().attn_tp_size
tp_rank = get_parallel().attn_tp_rank
ckpt_tp = expected_fused_tp_size
shards_per_rank = ckpt_tp // tp_size
for scale_name, ckpt_scale in deferred_scale_inv.items():
weight_name = scale_name.replace(".weight_scale_inv", ".weight")
@@ -208,6 +212,9 @@ def _resolve_deferred_qkv_scale_inv(
f"weight {weight_name} not found"
)
qkv_proj = unwrap_lora_layer(model.get_submodule(weight_name.rsplit(".", 1)[0]))
tp_rank, tp_size = get_group_rank_size(qkv_proj.tp_group)
shards_per_rank = ckpt_tp // tp_size
weight_param = params_dict[weight_name]
scale_param = params_dict[scale_name]
weight_data = weight_param.data
@@ -955,6 +962,7 @@ class MiMoV2Model(nn.Module):
self.padding_idx = getattr(config, "pad_token_id", None)
self.vocab_size = config.vocab_size
self.pp_group = get_parallel().pp_group
self._kv_cache_parallel_layout = resolve_linear_parallel_group("attn_tp")
self.layers_to_capture = []
if self.pp_group.is_first_rank:
@@ -1078,8 +1086,7 @@ class MiMoV2Model(nn.Module):
# factors (or else raise an exception). Thus, handled exceptions should
# make sure to leave KV cache scale factors in a known good (dummy) state
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
attn_tp_rank = get_parallel().attn_tp_rank
attn_tp_size = get_parallel().attn_tp_size
attn_tp_rank, attn_tp_size = self._kv_cache_parallel_layout
for layer_idx, scaling_factor in kv_cache_scales_loader(
quantization_param_path,
attn_tp_rank,
@@ -1546,6 +1553,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
loaded_weight,
expected_fused_tp_size,
deferred_scale_inv=deferred_qkv_scale_inv,
qkv_proj=self.get_submodule(name.rsplit(".", 1)[0]),
)
continue
@@ -1598,7 +1606,13 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
if name in params_dict.keys():
param = params_dict[name]
if "attention_sink_bias" in name:
start = get_parallel().attn_tp_rank * param.numel()
projection = unwrap_lora_layer(
self.get_submodule(name.rsplit(".", 1)[0]).qkv_proj
)
start = (
get_group_rank_size(projection.tp_group)[0]
* param.numel()
)
param.data.copy_(
loaded_weight[start : start + param.numel()]
)
@@ -1617,6 +1631,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
deferred_qkv_scale_inv,
expected_fused_tp_size,
config=self.config,
model=self,
)
def get_embed_and_head(self):
+9 -1
View File
@@ -20,6 +20,7 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -39,6 +40,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.mimo_v2 import (
@@ -308,6 +310,7 @@ class MiMoV2MTP(MiMoV2ForCausalLM):
expected_fused_tp_size=get_mimo_v2_fused_qkv_expected_tp_size(
self.config
),
qkv_proj=self.get_submodule(name.rsplit(".", 1)[0]),
)
continue
@@ -341,7 +344,12 @@ class MiMoV2MTP(MiMoV2ForCausalLM):
if name in params_dict.keys():
param = params_dict[name]
if "attention_sink_bias" in name:
start = get_parallel().attn_tp_rank * param.numel()
projection = unwrap_lora_layer(
self.get_submodule(name.rsplit(".", 1)[0]).qkv_proj
)
start = (
get_group_rank_size(projection.tp_group)[0] * param.numel()
)
param.data.copy_(loaded_weight[start : start + param.numel()])
else:
weight_loader = getattr(
+3 -2
View File
@@ -27,6 +27,7 @@ from sglang.srt.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
from sglang.srt.layers.pooler import Pooler, PoolingType
@@ -329,6 +330,7 @@ class OPTModel(nn.Module):
self.padding_idx = config.pad_token_id
self.vocab_size = config.vocab_size
self.pp_group = get_parallel().pp_group
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
self.decoder = OPTDecoder(
config=config,
@@ -353,8 +355,7 @@ class OPTModel(nn.Module):
)
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
tp_size = get_parallel().tp_size
tp_rank = get_parallel().tp_rank
tp_rank, tp_size = self._kv_cache_parallel_layout
for layer_idx, scaling_factor in kv_cache_scales_loader(
quantization_param_path,
tp_rank,
+3 -2
View File
@@ -40,6 +40,7 @@ from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.pooler import Pooler, PoolingType
@@ -342,6 +343,7 @@ class Qwen2Model(nn.Module):
self.padding_idx = getattr(config, "pad_token_id", None)
self.vocab_size = config.vocab_size
self.pp_group = get_parallel().pp_group
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
if self.pp_group.is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
@@ -454,8 +456,7 @@ class Qwen2Model(nn.Module):
# factors (or else raise an exception). Thus, handled exceptions should
# make sure to leave KV cache scale factors in a known good (dummy) state
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
tp_size = get_parallel().tp_size
tp_rank = get_parallel().tp_rank
tp_rank, tp_size = self._kv_cache_parallel_layout
for layer_idx, scaling_factor in kv_cache_scales_loader(
quantization_param_path,
tp_rank,
+3 -2
View File
@@ -14,6 +14,7 @@ from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
from sglang.srt.layers.quantization import QuantizationConfig
@@ -293,6 +294,7 @@ class SolarModel(nn.Module):
self.vocab_size = config.vocab_size
self.org_vocab_size = config.vocab_size
self.pp_group = get_parallel().pp_group
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
if self.pp_group.is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
@@ -381,8 +383,7 @@ class SolarModel(nn.Module):
return hidden_states
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
tp_size = get_parallel().tp_size
tp_rank = get_parallel().tp_rank
tp_rank, tp_size = self._kv_cache_parallel_layout
for layer_idx, scaling_factor in kv_cache_scales_loader(
quantization_param_path,
tp_rank,
@@ -76,11 +76,21 @@ def _weight_or_none(module: Optional[nn.Module]) -> Optional[torch.Tensor]:
def resolve_target_embed_and_head(
target_model: nn.Module,
target_model: nn.Module, *, draft_model: Optional[nn.Module] = None
) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
"""PP-safe getter: a PPMissingLayer embedding maps to ``embed=None``; any
other AttributeError propagates."""
try:
if draft_model is not None and hasattr(
target_model, "get_embed_and_head_for_draft"
):
found = _find_draft_embedding_module(draft_model, allow_replicated=True)
if found is None:
raise ValueError(
f"Draft model {draft_model.__class__.__name__} has no single "
"input embedding for target embedding sharing."
)
return target_model.get_embed_and_head_for_draft(found[1])
return target_model.get_embed_and_head()
except AttributeError:
if not _target_input_embedding_is_missing(target_model):
@@ -88,24 +98,38 @@ def resolve_target_embed_and_head(
return None, _weight_or_none(target_model.lm_head)
def find_draft_embedding_param(
draft_model: nn.Module,
) -> Optional[Tuple[str, nn.Parameter]]:
"""The draft's own input embedding: its single ``VocabParallelEmbedding``."""
def _find_draft_embedding_module(
draft_model: nn.Module, *, allow_replicated: bool = False
) -> Optional[Tuple[str, nn.Module]]:
"""Find a single vocab-parallel or named replicated input embedding."""
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
found = [
(f"{name}.weight", module.weight)
(f"{name}.weight", module)
for name, module in draft_model.named_modules()
if isinstance(module, VocabParallelEmbedding)
if (
isinstance(module, VocabParallelEmbedding)
or (
allow_replicated
and isinstance(module, nn.Embedding)
and name.rsplit(".", 1)[-1] in _EMBED_ATTR_NAMES
)
)
and not isinstance(module, ParallelLMHead)
]
return found[0] if len(found) == 1 else None
def find_draft_embedding_param(
draft_model: nn.Module,
) -> Optional[Tuple[str, nn.Parameter]]:
found = _find_draft_embedding_module(draft_model)
return (found[0], found[1].weight) if found is not None else None
def _is_input_embedding_key(key: str) -> bool:
if not key.endswith(_EMBED_KEY_SUFFIXES) or _LAYER_KEY_RE.search(key):
return False
@@ -233,7 +257,7 @@ def resolve_draft_embed_and_head(
) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
"""Embed/head to bind into a draft; a stage without the target embedding
loads the draft's own from the checkpoint."""
embed, head = resolve_target_embed_and_head(target_model)
embed, head = resolve_target_embed_and_head(target_model, draft_model=draft_model)
if embed is None:
embed = load_draft_embedding_from_checkpoint(
draft_model, model_path, revision=revision, load_config=load_config