[Refactor] Use destination layouts for model checkpoint mappings (#42590)
This commit is contained in:
@@ -27,6 +27,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import get_attn_tp_context
|
||||
from sglang.srt.layers.linear import _resolve_linear_group
|
||||
from sglang.srt.layers.parameter import BasevLLMParameter
|
||||
from sglang.srt.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
@@ -248,6 +249,11 @@ class VocabParallelEmbedding(torch.nn.Module):
|
||||
|
||||
self.enable_tp = enable_tp
|
||||
self.use_attn_tp_group = use_attn_tp_group
|
||||
self.tp_group = (
|
||||
_resolve_linear_group("attn_tp" if use_attn_tp_group else "tp")
|
||||
if enable_tp
|
||||
else None
|
||||
)
|
||||
if self.enable_tp:
|
||||
if use_attn_tp_group:
|
||||
tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
@@ -39,6 +39,7 @@ from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
@@ -311,6 +312,7 @@ class ApertusModel(nn.Module):
|
||||
self.vocab_size = config.vocab_size
|
||||
self.org_vocab_size = config.vocab_size
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
|
||||
if self.pp_group.is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
@@ -381,8 +383,7 @@ class ApertusModel(nn.Module):
|
||||
# factors (or else raise an exception). Thus, handled exceptions should
|
||||
# make sure to leave KV cache scale factors in a known good (dummy) state
|
||||
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
tp_rank, tp_size = self._kv_cache_parallel_layout
|
||||
for layer_idx, scaling_factor in kv_cache_scales_loader(
|
||||
quantization_param_path,
|
||||
tp_rank,
|
||||
|
||||
@@ -33,6 +33,7 @@ from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
@@ -285,6 +286,7 @@ class ArceeModel(nn.Module):
|
||||
self.padding_idx = config.pad_token_id
|
||||
self.vocab_size = config.vocab_size
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
|
||||
if self.pp_group.is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
@@ -352,8 +354,7 @@ class ArceeModel(nn.Module):
|
||||
return hidden_states, aux_hidden_states
|
||||
|
||||
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
tp_rank, tp_size = self._kv_cache_parallel_layout
|
||||
for layer_idx, scaling_factor in kv_cache_scales_loader(
|
||||
quantization_param_path,
|
||||
tp_rank,
|
||||
|
||||
@@ -92,6 +92,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternTokenPairs,
|
||||
general_mm_embed_routine,
|
||||
@@ -1937,12 +1938,17 @@ class Dots3LanguageModelForCausalLM(nn.Module):
|
||||
"Dots3 requires q_lora_rank to enable fused_qkv_a_g_proj_with_mqa loading."
|
||||
)
|
||||
cached_a_proj = {} if fuse_qkv_a_g_proj else None
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
def shard_g_proj_for_attention_tp(
|
||||
weight: torch.Tensor, cat_dim: int, is_scale: bool
|
||||
weight: torch.Tensor,
|
||||
cat_dim: int,
|
||||
is_scale: bool,
|
||||
*,
|
||||
q_b_proj: ColumnParallelLinear,
|
||||
):
|
||||
group = unwrap_lora_layer(q_b_proj).tp_group
|
||||
attn_tp_rank = group.rank_in_group if group is not None else 0
|
||||
attn_tp_size = group.world_size if group is not None else 1
|
||||
assert weight.ndim > cat_dim, (
|
||||
f"weight.ndim={weight.ndim}, cat_dim={cat_dim}"
|
||||
)
|
||||
@@ -1991,7 +1997,10 @@ class Dots3LanguageModelForCausalLM(nn.Module):
|
||||
kv_a_proj_weight = cached_a_proj[kv_a_proj_name]
|
||||
g_proj_weight = cached_a_proj[g_proj_name]
|
||||
g_proj_shard = shard_g_proj_for_attention_tp(
|
||||
g_proj_weight, cat_dim, is_scale
|
||||
g_proj_weight,
|
||||
cat_dim,
|
||||
is_scale,
|
||||
q_b_proj=self.get_submodule(param_name.rsplit(".", 2)[0]).q_b_proj,
|
||||
)
|
||||
scale_block_n = _get_scale_block_n(self.quant_config)
|
||||
kv_a_proj_weight_aligned = (
|
||||
|
||||
@@ -28,19 +28,24 @@ from transformers import (
|
||||
PreTrainedModel,
|
||||
)
|
||||
|
||||
from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.layers.activation import GeluAndMul
|
||||
from sglang.srt.layers.layernorm import Gemma3RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
_resolve_linear_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.pooler import EmbeddingPoolerOutput, Pooler, PoolingType
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.radix_attention import AttentionType, RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import apply_rotary_pos_emb, get_rope
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
@@ -735,6 +740,7 @@ class Gemma3ForCausalLM(PreTrainedModel):
|
||||
super().__init__(config=config)
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self._shared_vocab_tp_group = _resolve_linear_group("tp")
|
||||
self.model = Gemma3TextModel(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
@@ -919,28 +925,35 @@ class Gemma3ForCausalLM(PreTrainedModel):
|
||||
# of the (i-1)th layer as aux hidden state
|
||||
self.model.layers_to_capture = [val + 1 for val in layer_ids]
|
||||
|
||||
def _shard_weight(self, weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Shard a full embedding/lm_head weight along vocab dim for the current TP rank.
|
||||
|
||||
Gemma3 uses nn.Embedding (unsharded) but the Eagle3 draft model uses
|
||||
VocabParallelEmbedding (sharded). This method extracts the correct
|
||||
shard so the weights can be shared.
|
||||
"""
|
||||
tp_size = get_parallel().tp_size
|
||||
def _shard_weight(
|
||||
self, weight: torch.Tensor, *, draft_embedding=None
|
||||
) -> torch.Tensor:
|
||||
group = self._shared_vocab_tp_group
|
||||
if draft_embedding is not None:
|
||||
group = (
|
||||
draft_embedding.tp_group
|
||||
if isinstance(draft_embedding, VocabParallelEmbedding)
|
||||
else None
|
||||
)
|
||||
tp_rank, tp_size = get_group_rank_size(group)
|
||||
if tp_size <= 1:
|
||||
return weight
|
||||
tp_rank = get_parallel().tp_rank
|
||||
shard_size = (weight.shape[0] + tp_size - 1) // tp_size
|
||||
return weight[tp_rank * shard_size : (tp_rank + 1) * shard_size]
|
||||
|
||||
def get_embed(self):
|
||||
return self._shard_weight(self.model.embed_tokens.weight)
|
||||
|
||||
def get_embed_and_head(self):
|
||||
embed = self._shard_weight(self.model.embed_tokens.weight)
|
||||
head = self._shard_weight(self.lm_head.weight)
|
||||
def get_embed_and_head(self, *, draft_embedding=None):
|
||||
embed = self._shard_weight(
|
||||
self.model.embed_tokens.weight, draft_embedding=draft_embedding
|
||||
)
|
||||
head = self._shard_weight(self.lm_head.weight, draft_embedding=draft_embedding)
|
||||
return embed, head
|
||||
|
||||
def get_embed_and_head_for_draft(self, draft_embedding):
|
||||
return self.get_embed_and_head(draft_embedding=draft_embedding)
|
||||
|
||||
|
||||
class EmbeddingGemmaModel(Gemma3ForCausalLM):
|
||||
"""EmbeddingGemma's Gemma3 encoder with normalized mean pooling."""
|
||||
@@ -957,6 +970,7 @@ class EmbeddingGemmaModel(Gemma3ForCausalLM):
|
||||
PreTrainedModel.__init__(self, config=config)
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self._shared_vocab_tp_group = _resolve_linear_group("tp")
|
||||
self.model = Gemma3TextModel(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
|
||||
@@ -494,6 +494,9 @@ class Gemma3ForConditionalGeneration(PreTrainedModel):
|
||||
# For EAGLE3, head might not be needed
|
||||
return embed, None
|
||||
|
||||
def get_embed_and_head_for_draft(self, draft_embedding):
|
||||
return self.language_model.get_embed_and_head_for_draft(draft_embedding)
|
||||
|
||||
def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None):
|
||||
if hasattr(self.language_model, "set_eagle3_layers_to_capture"):
|
||||
self.language_model.set_eagle3_layers_to_capture(layer_ids)
|
||||
|
||||
@@ -33,6 +33,7 @@ from sglang.kernels.ops.moe.gemma4_routing import (
|
||||
gemma4_fused_routing,
|
||||
gemma_routing_post_topk,
|
||||
)
|
||||
from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
SumGroup,
|
||||
@@ -48,6 +49,7 @@ from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
_resolve_linear_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
@@ -57,7 +59,10 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
from sglang.srt.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
@@ -1131,6 +1136,7 @@ class Gemma4ForCausalLM(PreTrainedModel):
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self._shared_vocab_tp_group = _resolve_linear_group("tp")
|
||||
|
||||
self.model = Gemma4TextModel(
|
||||
config=config, quant_config=quant_config, prefix=add_prefix("model", prefix)
|
||||
@@ -1425,24 +1431,26 @@ class Gemma4ForCausalLM(PreTrainedModel):
|
||||
logger.log(level, "%s: %s", msg, names)
|
||||
return loaded_params
|
||||
|
||||
def _shard_weight(self, weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Shard a full embedding/lm_head weight along vocab dim for the current TP rank.
|
||||
|
||||
Gemma4 uses nn.Embedding (unsharded) but the Eagle3 draft model uses
|
||||
VocabParallelEmbedding (sharded). This method extracts the correct
|
||||
shard so the weights can be shared.
|
||||
"""
|
||||
tp_size = get_parallel().tp_size
|
||||
def _shard_weight(
|
||||
self, weight: torch.Tensor, *, draft_embedding=None
|
||||
) -> torch.Tensor:
|
||||
group = self._shared_vocab_tp_group
|
||||
if draft_embedding is not None:
|
||||
group = (
|
||||
draft_embedding.tp_group
|
||||
if isinstance(draft_embedding, VocabParallelEmbedding)
|
||||
else None
|
||||
)
|
||||
tp_rank, tp_size = get_group_rank_size(group)
|
||||
if tp_size <= 1:
|
||||
return weight
|
||||
tp_rank = get_parallel().tp_rank
|
||||
shard_size = (weight.shape[0] + tp_size - 1) // tp_size
|
||||
return weight[tp_rank * shard_size : (tp_rank + 1) * shard_size]
|
||||
|
||||
def get_embed(self):
|
||||
return self._shard_weight(self.model.embed_tokens.weight)
|
||||
|
||||
def get_embed_and_head(self):
|
||||
def get_embed_and_head(self, *, draft_embedding=None):
|
||||
if self.pp_group.world_size > 1:
|
||||
# Under PP, embed_tokens lives on the first rank and lm_head on
|
||||
# the last; neither rank holds both tensors, so we can't return
|
||||
@@ -1456,10 +1464,15 @@ class Gemma4ForCausalLM(PreTrainedModel):
|
||||
"PP rank and lm_head on the last; use --pp-size 1 if you "
|
||||
"need this API."
|
||||
)
|
||||
embed = self._shard_weight(self.model.embed_tokens.weight)
|
||||
head = self._shard_weight(self.lm_head.weight)
|
||||
embed = self._shard_weight(
|
||||
self.model.embed_tokens.weight, draft_embedding=draft_embedding
|
||||
)
|
||||
head = self._shard_weight(self.lm_head.weight, draft_embedding=draft_embedding)
|
||||
return embed, head
|
||||
|
||||
def get_embed_and_head_for_draft(self, draft_embedding):
|
||||
return self.get_embed_and_head(draft_embedding=draft_embedding)
|
||||
|
||||
def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None):
|
||||
if layer_ids is None:
|
||||
self.capture_aux_hidden_states = True
|
||||
|
||||
@@ -38,6 +38,7 @@ from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
@@ -318,6 +319,7 @@ class Glm4Model(nn.Module):
|
||||
self.padding_idx = config.pad_token_id
|
||||
self.vocab_size = config.vocab_size
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
|
||||
|
||||
if self.pp_group.is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
@@ -399,8 +401,7 @@ class Glm4Model(nn.Module):
|
||||
# factors (or else raise an exception). Thus, handled exceptions should
|
||||
# make sure to leave KV cache scale factors in a known good (dummy) state
|
||||
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
tp_rank, tp_size = self._kv_cache_parallel_layout
|
||||
for layer_idx, scaling_factor in kv_cache_scales_loader(
|
||||
quantization_param_path,
|
||||
tp_rank,
|
||||
|
||||
@@ -27,6 +27,7 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
@@ -62,6 +63,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
|
||||
get_tc_piecewise_forward_context,
|
||||
@@ -1274,8 +1276,11 @@ class GptOssForCausalLM(nn.Module):
|
||||
if name in params_dict.keys():
|
||||
param = params_dict[name]
|
||||
if "sinks" in name:
|
||||
start = get_parallel().attn_tp_rank * param.numel()
|
||||
tp_size = get_parallel().tp_size
|
||||
projection = unwrap_lora_layer(
|
||||
self.get_submodule(name.rsplit(".", 1)[0]).qkv_proj
|
||||
)
|
||||
tp_rank, tp_size = get_group_rank_size(projection.tp_group)
|
||||
start = tp_rank * param.numel()
|
||||
full_shard_size = param.numel() * tp_size
|
||||
# This handles TP padding: if the checkpoint dim is not divisible by tp_size,
|
||||
# the last TP shard extends beyond `loaded_weight`, pad with zeros before slicing.
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
@@ -573,6 +574,7 @@ class HunYuanMoEV1ForCausalLM(nn.Module):
|
||||
super().__init__()
|
||||
|
||||
self.config = config
|
||||
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
|
||||
|
||||
self.model = HunYuanModel(config, quant_config, prefix="model")
|
||||
self.unpadded_vocab_size = config.vocab_size
|
||||
@@ -769,8 +771,7 @@ class HunYuanMoEV1ForCausalLM(nn.Module):
|
||||
# factors (or else raise an exception). Thus, handled exceptions should
|
||||
# make sure to leave KV cache scale factors in a known good (dummy) state
|
||||
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
tp_rank, tp_size = self._kv_cache_parallel_layout
|
||||
for layer_idx, scaling_factor in kv_cache_scales_loader(
|
||||
quantization_param_path,
|
||||
tp_rank,
|
||||
|
||||
@@ -5,9 +5,11 @@ from typing import Any, Dict, Iterable, Optional, Tuple
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.qwen3_moe import Qwen3MoeAttention, Qwen3MoeDecoderLayer
|
||||
@@ -15,7 +17,6 @@ from sglang.srt.models.qwen3_vl_moe import (
|
||||
Qwen3MoeLLMModel,
|
||||
Qwen3VLMoeForConditionalGeneration,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -211,8 +212,8 @@ class InternS1ProForConditionalGeneration(Qwen3VLMoeForConditionalGeneration):
|
||||
|
||||
def _load_fope_weights(self, name: str, loaded_weight: torch.Tensor, params_dict):
|
||||
"""load fope weights"""
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
qkv_proj = unwrap_lora_layer(self.model.layers[0].self_attn.qkv_proj)
|
||||
attn_tp_rank, attn_tp_size = get_group_rank_size(qkv_proj.tp_group)
|
||||
|
||||
num_key_value_heads = loaded_weight.size(0)
|
||||
# replicate head if necessary
|
||||
|
||||
@@ -43,6 +43,7 @@ from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
@@ -406,6 +407,7 @@ class LlamaModel(nn.Module):
|
||||
self.padding_idx = config.pad_token_id
|
||||
self.vocab_size = config.vocab_size
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
|
||||
if self.pp_group.is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
@@ -485,8 +487,7 @@ class LlamaModel(nn.Module):
|
||||
# factors (or else raise an exception). Thus, handled exceptions should
|
||||
# make sure to leave KV cache scale factors in a known good (dummy) state
|
||||
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
tp_rank, tp_size = self._kv_cache_parallel_layout
|
||||
for layer_idx, scaling_factor in kv_cache_scales_loader(
|
||||
quantization_param_path,
|
||||
tp_rank,
|
||||
|
||||
@@ -24,6 +24,7 @@ from torch import nn
|
||||
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
|
||||
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
|
||||
from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -48,6 +49,7 @@ from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
@@ -65,6 +67,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
@@ -101,9 +104,11 @@ def load_mimo_v2_qkv_proj_weight(
|
||||
loaded_weight,
|
||||
expected_fused_tp_size: Optional[int] = None,
|
||||
deferred_scale_inv: Optional[Dict[str, torch.Tensor]] = None,
|
||||
*,
|
||||
qkv_proj: QKVParallelLinear,
|
||||
):
|
||||
tp_size = get_parallel().attn_tp_size
|
||||
tp_rank = get_parallel().attn_tp_rank
|
||||
qkv_proj = unwrap_lora_layer(qkv_proj)
|
||||
tp_rank, tp_size = get_group_rank_size(qkv_proj.tp_group)
|
||||
ckpt_tp = expected_fused_tp_size if expected_fused_tp_size is not None else tp_size
|
||||
|
||||
if ckpt_tp == tp_size and loaded_weight.shape == param.shape:
|
||||
@@ -194,11 +199,10 @@ def _resolve_deferred_qkv_scale_inv(
|
||||
expected_fused_tp_size: int,
|
||||
block_size: int = 128,
|
||||
config=None,
|
||||
*,
|
||||
model: nn.Module,
|
||||
):
|
||||
tp_size = get_parallel().attn_tp_size
|
||||
tp_rank = get_parallel().attn_tp_rank
|
||||
ckpt_tp = expected_fused_tp_size
|
||||
shards_per_rank = ckpt_tp // tp_size
|
||||
|
||||
for scale_name, ckpt_scale in deferred_scale_inv.items():
|
||||
weight_name = scale_name.replace(".weight_scale_inv", ".weight")
|
||||
@@ -208,6 +212,9 @@ def _resolve_deferred_qkv_scale_inv(
|
||||
f"weight {weight_name} not found"
|
||||
)
|
||||
|
||||
qkv_proj = unwrap_lora_layer(model.get_submodule(weight_name.rsplit(".", 1)[0]))
|
||||
tp_rank, tp_size = get_group_rank_size(qkv_proj.tp_group)
|
||||
shards_per_rank = ckpt_tp // tp_size
|
||||
weight_param = params_dict[weight_name]
|
||||
scale_param = params_dict[scale_name]
|
||||
weight_data = weight_param.data
|
||||
@@ -955,6 +962,7 @@ class MiMoV2Model(nn.Module):
|
||||
self.padding_idx = getattr(config, "pad_token_id", None)
|
||||
self.vocab_size = config.vocab_size
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self._kv_cache_parallel_layout = resolve_linear_parallel_group("attn_tp")
|
||||
self.layers_to_capture = []
|
||||
|
||||
if self.pp_group.is_first_rank:
|
||||
@@ -1078,8 +1086,7 @@ class MiMoV2Model(nn.Module):
|
||||
# factors (or else raise an exception). Thus, handled exceptions should
|
||||
# make sure to leave KV cache scale factors in a known good (dummy) state
|
||||
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
attn_tp_rank, attn_tp_size = self._kv_cache_parallel_layout
|
||||
for layer_idx, scaling_factor in kv_cache_scales_loader(
|
||||
quantization_param_path,
|
||||
attn_tp_rank,
|
||||
@@ -1546,6 +1553,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
|
||||
loaded_weight,
|
||||
expected_fused_tp_size,
|
||||
deferred_scale_inv=deferred_qkv_scale_inv,
|
||||
qkv_proj=self.get_submodule(name.rsplit(".", 1)[0]),
|
||||
)
|
||||
continue
|
||||
|
||||
@@ -1598,7 +1606,13 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
|
||||
if name in params_dict.keys():
|
||||
param = params_dict[name]
|
||||
if "attention_sink_bias" in name:
|
||||
start = get_parallel().attn_tp_rank * param.numel()
|
||||
projection = unwrap_lora_layer(
|
||||
self.get_submodule(name.rsplit(".", 1)[0]).qkv_proj
|
||||
)
|
||||
start = (
|
||||
get_group_rank_size(projection.tp_group)[0]
|
||||
* param.numel()
|
||||
)
|
||||
param.data.copy_(
|
||||
loaded_weight[start : start + param.numel()]
|
||||
)
|
||||
@@ -1617,6 +1631,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
|
||||
deferred_qkv_scale_inv,
|
||||
expected_fused_tp_size,
|
||||
config=self.config,
|
||||
model=self,
|
||||
)
|
||||
|
||||
def get_embed_and_head(self):
|
||||
|
||||
@@ -20,6 +20,7 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
|
||||
from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -39,6 +40,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.mimo_v2 import (
|
||||
@@ -308,6 +310,7 @@ class MiMoV2MTP(MiMoV2ForCausalLM):
|
||||
expected_fused_tp_size=get_mimo_v2_fused_qkv_expected_tp_size(
|
||||
self.config
|
||||
),
|
||||
qkv_proj=self.get_submodule(name.rsplit(".", 1)[0]),
|
||||
)
|
||||
continue
|
||||
|
||||
@@ -341,7 +344,12 @@ class MiMoV2MTP(MiMoV2ForCausalLM):
|
||||
if name in params_dict.keys():
|
||||
param = params_dict[name]
|
||||
if "attention_sink_bias" in name:
|
||||
start = get_parallel().attn_tp_rank * param.numel()
|
||||
projection = unwrap_lora_layer(
|
||||
self.get_submodule(name.rsplit(".", 1)[0]).qkv_proj
|
||||
)
|
||||
start = (
|
||||
get_group_rank_size(projection.tp_group)[0] * param.numel()
|
||||
)
|
||||
param.data.copy_(loaded_weight[start : start + param.numel()])
|
||||
else:
|
||||
weight_loader = getattr(
|
||||
|
||||
@@ -27,6 +27,7 @@ from sglang.srt.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
@@ -329,6 +330,7 @@ class OPTModel(nn.Module):
|
||||
self.padding_idx = config.pad_token_id
|
||||
self.vocab_size = config.vocab_size
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
|
||||
|
||||
self.decoder = OPTDecoder(
|
||||
config=config,
|
||||
@@ -353,8 +355,7 @@ class OPTModel(nn.Module):
|
||||
)
|
||||
|
||||
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
tp_rank, tp_size = self._kv_cache_parallel_layout
|
||||
for layer_idx, scaling_factor in kv_cache_scales_loader(
|
||||
quantization_param_path,
|
||||
tp_rank,
|
||||
|
||||
@@ -40,6 +40,7 @@ from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
@@ -342,6 +343,7 @@ class Qwen2Model(nn.Module):
|
||||
self.padding_idx = getattr(config, "pad_token_id", None)
|
||||
self.vocab_size = config.vocab_size
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
|
||||
|
||||
if self.pp_group.is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
@@ -454,8 +456,7 @@ class Qwen2Model(nn.Module):
|
||||
# factors (or else raise an exception). Thus, handled exceptions should
|
||||
# make sure to leave KV cache scale factors in a known good (dummy) state
|
||||
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
tp_rank, tp_size = self._kv_cache_parallel_layout
|
||||
for layer_idx, scaling_factor in kv_cache_scales_loader(
|
||||
quantization_param_path,
|
||||
tp_rank,
|
||||
|
||||
@@ -14,6 +14,7 @@ from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.quantization import QuantizationConfig
|
||||
@@ -293,6 +294,7 @@ class SolarModel(nn.Module):
|
||||
self.vocab_size = config.vocab_size
|
||||
self.org_vocab_size = config.vocab_size
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp")
|
||||
if self.pp_group.is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
@@ -381,8 +383,7 @@ class SolarModel(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
tp_rank, tp_size = self._kv_cache_parallel_layout
|
||||
for layer_idx, scaling_factor in kv_cache_scales_loader(
|
||||
quantization_param_path,
|
||||
tp_rank,
|
||||
|
||||
@@ -76,11 +76,21 @@ def _weight_or_none(module: Optional[nn.Module]) -> Optional[torch.Tensor]:
|
||||
|
||||
|
||||
def resolve_target_embed_and_head(
|
||||
target_model: nn.Module,
|
||||
target_model: nn.Module, *, draft_model: Optional[nn.Module] = None
|
||||
) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
|
||||
"""PP-safe getter: a PPMissingLayer embedding maps to ``embed=None``; any
|
||||
other AttributeError propagates."""
|
||||
try:
|
||||
if draft_model is not None and hasattr(
|
||||
target_model, "get_embed_and_head_for_draft"
|
||||
):
|
||||
found = _find_draft_embedding_module(draft_model, allow_replicated=True)
|
||||
if found is None:
|
||||
raise ValueError(
|
||||
f"Draft model {draft_model.__class__.__name__} has no single "
|
||||
"input embedding for target embedding sharing."
|
||||
)
|
||||
return target_model.get_embed_and_head_for_draft(found[1])
|
||||
return target_model.get_embed_and_head()
|
||||
except AttributeError:
|
||||
if not _target_input_embedding_is_missing(target_model):
|
||||
@@ -88,24 +98,38 @@ def resolve_target_embed_and_head(
|
||||
return None, _weight_or_none(target_model.lm_head)
|
||||
|
||||
|
||||
def find_draft_embedding_param(
|
||||
draft_model: nn.Module,
|
||||
) -> Optional[Tuple[str, nn.Parameter]]:
|
||||
"""The draft's own input embedding: its single ``VocabParallelEmbedding``."""
|
||||
def _find_draft_embedding_module(
|
||||
draft_model: nn.Module, *, allow_replicated: bool = False
|
||||
) -> Optional[Tuple[str, nn.Module]]:
|
||||
"""Find a single vocab-parallel or named replicated input embedding."""
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
|
||||
found = [
|
||||
(f"{name}.weight", module.weight)
|
||||
(f"{name}.weight", module)
|
||||
for name, module in draft_model.named_modules()
|
||||
if isinstance(module, VocabParallelEmbedding)
|
||||
if (
|
||||
isinstance(module, VocabParallelEmbedding)
|
||||
or (
|
||||
allow_replicated
|
||||
and isinstance(module, nn.Embedding)
|
||||
and name.rsplit(".", 1)[-1] in _EMBED_ATTR_NAMES
|
||||
)
|
||||
)
|
||||
and not isinstance(module, ParallelLMHead)
|
||||
]
|
||||
return found[0] if len(found) == 1 else None
|
||||
|
||||
|
||||
def find_draft_embedding_param(
|
||||
draft_model: nn.Module,
|
||||
) -> Optional[Tuple[str, nn.Parameter]]:
|
||||
found = _find_draft_embedding_module(draft_model)
|
||||
return (found[0], found[1].weight) if found is not None else None
|
||||
|
||||
|
||||
def _is_input_embedding_key(key: str) -> bool:
|
||||
if not key.endswith(_EMBED_KEY_SUFFIXES) or _LAYER_KEY_RE.search(key):
|
||||
return False
|
||||
@@ -233,7 +257,7 @@ def resolve_draft_embed_and_head(
|
||||
) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
|
||||
"""Embed/head to bind into a draft; a stage without the target embedding
|
||||
loads the draft's own from the checkpoint."""
|
||||
embed, head = resolve_target_embed_and_head(target_model)
|
||||
embed, head = resolve_target_embed_and_head(target_model, draft_model=draft_model)
|
||||
if embed is None:
|
||||
embed = load_draft_embedding_from_checkpoint(
|
||||
draft_model, model_path, revision=revision, load_config=load_config
|
||||
|
||||
Reference in New Issue
Block a user