From 7760d5264ac16b85392aefb1bf5c13ce38d9632e Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:24:22 -0700 Subject: [PATCH] [Refactor] Use destination layouts for model checkpoint mappings (#42590) --- .../srt/layers/vocab_parallel_embedding.py | 6 + python/sglang/srt/models/apertus.py | 5 +- python/sglang/srt/models/arcee.py | 5 +- .../srt/models/dots3_common/modeling.py | 17 +- python/sglang/srt/models/gemma3_causal.py | 40 ++- python/sglang/srt/models/gemma3_mm.py | 3 + python/sglang/srt/models/gemma4_causal.py | 39 ++- python/sglang/srt/models/glm4.py | 5 +- python/sglang/srt/models/gpt_oss.py | 9 +- python/sglang/srt/models/hunyuan.py | 5 +- python/sglang/srt/models/interns1pro.py | 7 +- python/sglang/srt/models/llama.py | 5 +- python/sglang/srt/models/mimo_v2.py | 31 +- python/sglang/srt/models/mimo_v2_nextn.py | 10 +- python/sglang/srt/models/opt.py | 5 +- python/sglang/srt/models/qwen2.py | 5 +- python/sglang/srt/models/solar.py | 5 +- .../srt/speculative/pp_draft_embedding.py | 40 ++- .../unit/dllm/test_gemma4_cuda_graph.py | 8 +- .../test_attention_sink_loader_layout.py | 210 ++++++++++++ .../test_dots_fused_gate_loader_layout.py | 272 ++++++++++++++++ .../models/test_gemma_shared_vocab_layout.py | 304 ++++++++++++++++++ .../test_interns1pro_fope_loader_layout.py | 197 ++++++++++++ .../test_kv_calibration_loader_layout.py | 261 +++++++++++++++ .../test_mimo_fused_qkv_loader_layout.py | 303 +++++++++++++++++ 25 files changed, 1725 insertions(+), 72 deletions(-) create mode 100644 test/registered/unit/models/test_attention_sink_loader_layout.py create mode 100644 test/registered/unit/models/test_dots_fused_gate_loader_layout.py create mode 100644 test/registered/unit/models/test_gemma_shared_vocab_layout.py create mode 100644 test/registered/unit/models/test_interns1pro_fope_loader_layout.py create mode 100644 test/registered/unit/models/test_kv_calibration_loader_layout.py create mode 100644 test/registered/unit/models/test_mimo_fused_qkv_loader_layout.py diff --git a/python/sglang/srt/layers/vocab_parallel_embedding.py b/python/sglang/srt/layers/vocab_parallel_embedding.py index 66fee24422..4e7a955451 100644 --- a/python/sglang/srt/layers/vocab_parallel_embedding.py +++ b/python/sglang/srt/layers/vocab_parallel_embedding.py @@ -27,6 +27,7 @@ from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) from sglang.srt.layers.layer_boundary import get_attn_tp_context +from sglang.srt.layers.linear import _resolve_linear_group from sglang.srt.layers.parameter import BasevLLMParameter from sglang.srt.layers.quantization.base_config import ( QuantizationConfig, @@ -248,6 +249,11 @@ class VocabParallelEmbedding(torch.nn.Module): self.enable_tp = enable_tp self.use_attn_tp_group = use_attn_tp_group + self.tp_group = ( + _resolve_linear_group("attn_tp" if use_attn_tp_group else "tp") + if enable_tp + else None + ) if self.enable_tp: if use_attn_tp_group: tp_rank = get_parallel().attn_tp_rank diff --git a/python/sglang/srt/models/apertus.py b/python/sglang/srt/models/apertus.py index 0414fb5adf..641cfbd68f 100644 --- a/python/sglang/srt/models/apertus.py +++ b/python/sglang/srt/models/apertus.py @@ -39,6 +39,7 @@ from sglang.srt.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput from sglang.srt.layers.pooler import Pooler, PoolingType @@ -311,6 +312,7 @@ class ApertusModel(nn.Module): self.vocab_size = config.vocab_size self.org_vocab_size = config.vocab_size self.pp_group = get_parallel().pp_group + self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp") if self.pp_group.is_first_rank: self.embed_tokens = VocabParallelEmbedding( config.vocab_size, @@ -381,8 +383,7 @@ class ApertusModel(nn.Module): # factors (or else raise an exception). Thus, handled exceptions should # make sure to leave KV cache scale factors in a known good (dummy) state def load_kv_cache_scales(self, quantization_param_path: str) -> None: - tp_size = get_parallel().tp_size - tp_rank = get_parallel().tp_rank + tp_rank, tp_size = self._kv_cache_parallel_layout for layer_idx, scaling_factor in kv_cache_scales_loader( quantization_param_path, tp_rank, diff --git a/python/sglang/srt/models/arcee.py b/python/sglang/srt/models/arcee.py index 9ee234e013..1624f18f46 100644 --- a/python/sglang/srt/models/arcee.py +++ b/python/sglang/srt/models/arcee.py @@ -33,6 +33,7 @@ from sglang.srt.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput from sglang.srt.layers.pooler import Pooler, PoolingType @@ -285,6 +286,7 @@ class ArceeModel(nn.Module): self.padding_idx = config.pad_token_id self.vocab_size = config.vocab_size self.pp_group = get_parallel().pp_group + self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp") if self.pp_group.is_first_rank: self.embed_tokens = VocabParallelEmbedding( config.vocab_size, @@ -352,8 +354,7 @@ class ArceeModel(nn.Module): return hidden_states, aux_hidden_states def load_kv_cache_scales(self, quantization_param_path: str) -> None: - tp_size = get_parallel().tp_size - tp_rank = get_parallel().tp_rank + tp_rank, tp_size = self._kv_cache_parallel_layout for layer_idx, scaling_factor in kv_cache_scales_loader( quantization_param_path, tp_rank, diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 90fcd00ea2..a7c13da00e 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -92,6 +92,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) +from sglang.srt.lora.layers import unwrap_lora_layer from sglang.srt.managers.mm_utils import ( MultiModalityDataPaddingPatternTokenPairs, general_mm_embed_routine, @@ -1937,12 +1938,17 @@ class Dots3LanguageModelForCausalLM(nn.Module): "Dots3 requires q_lora_rank to enable fused_qkv_a_g_proj_with_mqa loading." ) cached_a_proj = {} if fuse_qkv_a_g_proj else None - attn_tp_rank = get_parallel().attn_tp_rank - attn_tp_size = get_parallel().attn_tp_size def shard_g_proj_for_attention_tp( - weight: torch.Tensor, cat_dim: int, is_scale: bool + weight: torch.Tensor, + cat_dim: int, + is_scale: bool, + *, + q_b_proj: ColumnParallelLinear, ): + group = unwrap_lora_layer(q_b_proj).tp_group + attn_tp_rank = group.rank_in_group if group is not None else 0 + attn_tp_size = group.world_size if group is not None else 1 assert weight.ndim > cat_dim, ( f"weight.ndim={weight.ndim}, cat_dim={cat_dim}" ) @@ -1991,7 +1997,10 @@ class Dots3LanguageModelForCausalLM(nn.Module): kv_a_proj_weight = cached_a_proj[kv_a_proj_name] g_proj_weight = cached_a_proj[g_proj_name] g_proj_shard = shard_g_proj_for_attention_tp( - g_proj_weight, cat_dim, is_scale + g_proj_weight, + cat_dim, + is_scale, + q_b_proj=self.get_submodule(param_name.rsplit(".", 2)[0]).q_b_proj, ) scale_block_n = _get_scale_block_n(self.quant_config) kv_a_proj_weight_aligned = ( diff --git a/python/sglang/srt/models/gemma3_causal.py b/python/sglang/srt/models/gemma3_causal.py index 210156d84e..2db9d031e4 100644 --- a/python/sglang/srt/models/gemma3_causal.py +++ b/python/sglang/srt/models/gemma3_causal.py @@ -28,19 +28,24 @@ from transformers import ( PreTrainedModel, ) +from sglang.srt.distributed.utils import get_group_rank_size from sglang.srt.layers.activation import GeluAndMul from sglang.srt.layers.layernorm import Gemma3RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, + _resolve_linear_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.pooler import EmbeddingPoolerOutput, Pooler, PoolingType from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import AttentionType, RadixAttention from sglang.srt.layers.rotary_embedding import apply_rotary_pos_emb, get_rope -from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead +from sglang.srt.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_loader.weight_utils import ( default_weight_loader, @@ -735,6 +740,7 @@ class Gemma3ForCausalLM(PreTrainedModel): super().__init__(config=config) self.config = config self.quant_config = quant_config + self._shared_vocab_tp_group = _resolve_linear_group("tp") self.model = Gemma3TextModel( config, quant_config, prefix=add_prefix("model", prefix) ) @@ -919,28 +925,35 @@ class Gemma3ForCausalLM(PreTrainedModel): # of the (i-1)th layer as aux hidden state self.model.layers_to_capture = [val + 1 for val in layer_ids] - def _shard_weight(self, weight: torch.Tensor) -> torch.Tensor: - """Shard a full embedding/lm_head weight along vocab dim for the current TP rank. - - Gemma3 uses nn.Embedding (unsharded) but the Eagle3 draft model uses - VocabParallelEmbedding (sharded). This method extracts the correct - shard so the weights can be shared. - """ - tp_size = get_parallel().tp_size + def _shard_weight( + self, weight: torch.Tensor, *, draft_embedding=None + ) -> torch.Tensor: + group = self._shared_vocab_tp_group + if draft_embedding is not None: + group = ( + draft_embedding.tp_group + if isinstance(draft_embedding, VocabParallelEmbedding) + else None + ) + tp_rank, tp_size = get_group_rank_size(group) if tp_size <= 1: return weight - tp_rank = get_parallel().tp_rank shard_size = (weight.shape[0] + tp_size - 1) // tp_size return weight[tp_rank * shard_size : (tp_rank + 1) * shard_size] def get_embed(self): return self._shard_weight(self.model.embed_tokens.weight) - def get_embed_and_head(self): - embed = self._shard_weight(self.model.embed_tokens.weight) - head = self._shard_weight(self.lm_head.weight) + def get_embed_and_head(self, *, draft_embedding=None): + embed = self._shard_weight( + self.model.embed_tokens.weight, draft_embedding=draft_embedding + ) + head = self._shard_weight(self.lm_head.weight, draft_embedding=draft_embedding) return embed, head + def get_embed_and_head_for_draft(self, draft_embedding): + return self.get_embed_and_head(draft_embedding=draft_embedding) + class EmbeddingGemmaModel(Gemma3ForCausalLM): """EmbeddingGemma's Gemma3 encoder with normalized mean pooling.""" @@ -957,6 +970,7 @@ class EmbeddingGemmaModel(Gemma3ForCausalLM): PreTrainedModel.__init__(self, config=config) self.config = config self.quant_config = quant_config + self._shared_vocab_tp_group = _resolve_linear_group("tp") self.model = Gemma3TextModel( config, quant_config, prefix=add_prefix("model", prefix) ) diff --git a/python/sglang/srt/models/gemma3_mm.py b/python/sglang/srt/models/gemma3_mm.py index 52e5740594..e6eb155527 100644 --- a/python/sglang/srt/models/gemma3_mm.py +++ b/python/sglang/srt/models/gemma3_mm.py @@ -494,6 +494,9 @@ class Gemma3ForConditionalGeneration(PreTrainedModel): # For EAGLE3, head might not be needed return embed, None + def get_embed_and_head_for_draft(self, draft_embedding): + return self.language_model.get_embed_and_head_for_draft(draft_embedding) + def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None): if hasattr(self.language_model, "set_eagle3_layers_to_capture"): self.language_model.set_eagle3_layers_to_capture(layer_ids) diff --git a/python/sglang/srt/models/gemma4_causal.py b/python/sglang/srt/models/gemma4_causal.py index 2a2b7afe08..67e6ee7429 100644 --- a/python/sglang/srt/models/gemma4_causal.py +++ b/python/sglang/srt/models/gemma4_causal.py @@ -33,6 +33,7 @@ from sglang.kernels.ops.moe.gemma4_routing import ( gemma4_fused_routing, gemma_routing_post_topk, ) +from sglang.srt.distributed.utils import get_group_rank_size from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.layer_boundary import ( SumGroup, @@ -48,6 +49,7 @@ from sglang.srt.layers.linear import ( QKVParallelLinear, ReplicatedLinear, RowParallelLinear, + _resolve_linear_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class @@ -57,7 +59,10 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope from sglang.srt.layers.utils import PPMissingLayer, get_layer_id -from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead +from sglang.srt.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_loader.weight_utils import ( default_weight_loader, @@ -1131,6 +1136,7 @@ class Gemma4ForCausalLM(PreTrainedModel): self.pp_group = get_parallel().pp_group self.config = config self.quant_config = quant_config + self._shared_vocab_tp_group = _resolve_linear_group("tp") self.model = Gemma4TextModel( config=config, quant_config=quant_config, prefix=add_prefix("model", prefix) @@ -1425,24 +1431,26 @@ class Gemma4ForCausalLM(PreTrainedModel): logger.log(level, "%s: %s", msg, names) return loaded_params - def _shard_weight(self, weight: torch.Tensor) -> torch.Tensor: - """Shard a full embedding/lm_head weight along vocab dim for the current TP rank. - - Gemma4 uses nn.Embedding (unsharded) but the Eagle3 draft model uses - VocabParallelEmbedding (sharded). This method extracts the correct - shard so the weights can be shared. - """ - tp_size = get_parallel().tp_size + def _shard_weight( + self, weight: torch.Tensor, *, draft_embedding=None + ) -> torch.Tensor: + group = self._shared_vocab_tp_group + if draft_embedding is not None: + group = ( + draft_embedding.tp_group + if isinstance(draft_embedding, VocabParallelEmbedding) + else None + ) + tp_rank, tp_size = get_group_rank_size(group) if tp_size <= 1: return weight - tp_rank = get_parallel().tp_rank shard_size = (weight.shape[0] + tp_size - 1) // tp_size return weight[tp_rank * shard_size : (tp_rank + 1) * shard_size] def get_embed(self): return self._shard_weight(self.model.embed_tokens.weight) - def get_embed_and_head(self): + def get_embed_and_head(self, *, draft_embedding=None): if self.pp_group.world_size > 1: # Under PP, embed_tokens lives on the first rank and lm_head on # the last; neither rank holds both tensors, so we can't return @@ -1456,10 +1464,15 @@ class Gemma4ForCausalLM(PreTrainedModel): "PP rank and lm_head on the last; use --pp-size 1 if you " "need this API." ) - embed = self._shard_weight(self.model.embed_tokens.weight) - head = self._shard_weight(self.lm_head.weight) + embed = self._shard_weight( + self.model.embed_tokens.weight, draft_embedding=draft_embedding + ) + head = self._shard_weight(self.lm_head.weight, draft_embedding=draft_embedding) return embed, head + def get_embed_and_head_for_draft(self, draft_embedding): + return self.get_embed_and_head(draft_embedding=draft_embedding) + def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None): if layer_ids is None: self.capture_aux_hidden_states = True diff --git a/python/sglang/srt/models/glm4.py b/python/sglang/srt/models/glm4.py index 15b4d9775e..8695046f0a 100644 --- a/python/sglang/srt/models/glm4.py +++ b/python/sglang/srt/models/glm4.py @@ -38,6 +38,7 @@ from sglang.srt.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.pooler import Pooler, PoolingType @@ -318,6 +319,7 @@ class Glm4Model(nn.Module): self.padding_idx = config.pad_token_id self.vocab_size = config.vocab_size self.pp_group = get_parallel().pp_group + self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp") if self.pp_group.is_first_rank: self.embed_tokens = VocabParallelEmbedding( @@ -399,8 +401,7 @@ class Glm4Model(nn.Module): # factors (or else raise an exception). Thus, handled exceptions should # make sure to leave KV cache scale factors in a known good (dummy) state def load_kv_cache_scales(self, quantization_param_path: str) -> None: - tp_size = get_parallel().tp_size - tp_rank = get_parallel().tp_rank + tp_rank, tp_size = self._kv_cache_parallel_layout for layer_idx, scaling_factor in kv_cache_scales_loader( quantization_param_path, tp_rank, diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 44228d2412..2dc60d7889 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -27,6 +27,7 @@ from torch import nn from transformers import PretrainedConfig from sglang.kernels.jit.utils import is_arch_support_pdl +from sglang.srt.distributed.utils import get_group_rank_size from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList @@ -62,6 +63,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) +from sglang.srt.lora.layers import unwrap_lora_layer from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import ( get_tc_piecewise_forward_context, @@ -1274,8 +1276,11 @@ class GptOssForCausalLM(nn.Module): if name in params_dict.keys(): param = params_dict[name] if "sinks" in name: - start = get_parallel().attn_tp_rank * param.numel() - tp_size = get_parallel().tp_size + projection = unwrap_lora_layer( + self.get_submodule(name.rsplit(".", 1)[0]).qkv_proj + ) + tp_rank, tp_size = get_group_rank_size(projection.tp_group) + start = tp_rank * param.numel() full_shard_size = param.numel() * tp_size # This handles TP padding: if the checkpoint dim is not divisible by tp_size, # the last TP shard extends beyond `loaded_weight`, pad with zeros before slicing. diff --git a/python/sglang/srt/models/hunyuan.py b/python/sglang/srt/models/hunyuan.py index 5649a42c21..65ae511e8e 100644 --- a/python/sglang/srt/models/hunyuan.py +++ b/python/sglang/srt/models/hunyuan.py @@ -32,6 +32,7 @@ from sglang.srt.layers.linear import ( QKVParallelLinear, ReplicatedLinear, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.fused_moe_triton import FusedMoE @@ -573,6 +574,7 @@ class HunYuanMoEV1ForCausalLM(nn.Module): super().__init__() self.config = config + self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp") self.model = HunYuanModel(config, quant_config, prefix="model") self.unpadded_vocab_size = config.vocab_size @@ -769,8 +771,7 @@ class HunYuanMoEV1ForCausalLM(nn.Module): # factors (or else raise an exception). Thus, handled exceptions should # make sure to leave KV cache scale factors in a known good (dummy) state def load_kv_cache_scales(self, quantization_param_path: str) -> None: - tp_size = get_parallel().tp_size - tp_rank = get_parallel().tp_rank + tp_rank, tp_size = self._kv_cache_parallel_layout for layer_idx, scaling_factor in kv_cache_scales_loader( quantization_param_path, tp_rank, diff --git a/python/sglang/srt/models/interns1pro.py b/python/sglang/srt/models/interns1pro.py index 0bc524452b..17851f46a0 100644 --- a/python/sglang/srt/models/interns1pro.py +++ b/python/sglang/srt/models/interns1pro.py @@ -5,9 +5,11 @@ from typing import Any, Dict, Iterable, Optional, Tuple import torch from transformers import PretrainedConfig +from sglang.srt.distributed.utils import get_group_rank_size from sglang.srt.layers.moe.topk import TopK from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.rotary_embedding import get_rope +from sglang.srt.lora.layers import unwrap_lora_layer from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.qwen3_moe import Qwen3MoeAttention, Qwen3MoeDecoderLayer @@ -15,7 +17,6 @@ from sglang.srt.models.qwen3_vl_moe import ( Qwen3MoeLLMModel, Qwen3VLMoeForConditionalGeneration, ) -from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import add_prefix logger = logging.getLogger(__name__) @@ -211,8 +212,8 @@ class InternS1ProForConditionalGeneration(Qwen3VLMoeForConditionalGeneration): def _load_fope_weights(self, name: str, loaded_weight: torch.Tensor, params_dict): """load fope weights""" - attn_tp_size = get_parallel().attn_tp_size - attn_tp_rank = get_parallel().attn_tp_rank + qkv_proj = unwrap_lora_layer(self.model.layers[0].self_attn.qkv_proj) + attn_tp_rank, attn_tp_size = get_group_rank_size(qkv_proj.tp_group) num_key_value_heads = loaded_weight.size(0) # replicate head if necessary diff --git a/python/sglang/srt/models/llama.py b/python/sglang/srt/models/llama.py index 84569c13e0..9329e0054e 100644 --- a/python/sglang/srt/models/llama.py +++ b/python/sglang/srt/models/llama.py @@ -43,6 +43,7 @@ from sglang.srt.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput from sglang.srt.layers.pooler import Pooler, PoolingType @@ -406,6 +407,7 @@ class LlamaModel(nn.Module): self.padding_idx = config.pad_token_id self.vocab_size = config.vocab_size self.pp_group = get_parallel().pp_group + self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp") if self.pp_group.is_first_rank: self.embed_tokens = VocabParallelEmbedding( config.vocab_size, @@ -485,8 +487,7 @@ class LlamaModel(nn.Module): # factors (or else raise an exception). Thus, handled exceptions should # make sure to leave KV cache scale factors in a known good (dummy) state def load_kv_cache_scales(self, quantization_param_path: str) -> None: - tp_size = get_parallel().tp_size - tp_rank = get_parallel().tp_rank + tp_rank, tp_size = self._kv_cache_parallel_layout for layer_idx, scaling_factor in kv_cache_scales_loader( quantization_param_path, tp_rank, diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 9d9b69769d..a45ddbb2c8 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -24,6 +24,7 @@ from torch import nn from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size +from sglang.srt.distributed.utils import get_group_rank_size from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -48,6 +49,7 @@ from sglang.srt.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( @@ -65,6 +67,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) +from sglang.srt.lora.layers import unwrap_lora_layer from sglang.srt.managers.mm_utils import ( MultiModalityDataPaddingPatternMultimodalTokens, general_mm_embed_routine, @@ -101,9 +104,11 @@ def load_mimo_v2_qkv_proj_weight( loaded_weight, expected_fused_tp_size: Optional[int] = None, deferred_scale_inv: Optional[Dict[str, torch.Tensor]] = None, + *, + qkv_proj: QKVParallelLinear, ): - tp_size = get_parallel().attn_tp_size - tp_rank = get_parallel().attn_tp_rank + qkv_proj = unwrap_lora_layer(qkv_proj) + tp_rank, tp_size = get_group_rank_size(qkv_proj.tp_group) ckpt_tp = expected_fused_tp_size if expected_fused_tp_size is not None else tp_size if ckpt_tp == tp_size and loaded_weight.shape == param.shape: @@ -194,11 +199,10 @@ def _resolve_deferred_qkv_scale_inv( expected_fused_tp_size: int, block_size: int = 128, config=None, + *, + model: nn.Module, ): - tp_size = get_parallel().attn_tp_size - tp_rank = get_parallel().attn_tp_rank ckpt_tp = expected_fused_tp_size - shards_per_rank = ckpt_tp // tp_size for scale_name, ckpt_scale in deferred_scale_inv.items(): weight_name = scale_name.replace(".weight_scale_inv", ".weight") @@ -208,6 +212,9 @@ def _resolve_deferred_qkv_scale_inv( f"weight {weight_name} not found" ) + qkv_proj = unwrap_lora_layer(model.get_submodule(weight_name.rsplit(".", 1)[0])) + tp_rank, tp_size = get_group_rank_size(qkv_proj.tp_group) + shards_per_rank = ckpt_tp // tp_size weight_param = params_dict[weight_name] scale_param = params_dict[scale_name] weight_data = weight_param.data @@ -955,6 +962,7 @@ class MiMoV2Model(nn.Module): self.padding_idx = getattr(config, "pad_token_id", None) self.vocab_size = config.vocab_size self.pp_group = get_parallel().pp_group + self._kv_cache_parallel_layout = resolve_linear_parallel_group("attn_tp") self.layers_to_capture = [] if self.pp_group.is_first_rank: @@ -1078,8 +1086,7 @@ class MiMoV2Model(nn.Module): # factors (or else raise an exception). Thus, handled exceptions should # make sure to leave KV cache scale factors in a known good (dummy) state def load_kv_cache_scales(self, quantization_param_path: str) -> None: - attn_tp_rank = get_parallel().attn_tp_rank - attn_tp_size = get_parallel().attn_tp_size + attn_tp_rank, attn_tp_size = self._kv_cache_parallel_layout for layer_idx, scaling_factor in kv_cache_scales_loader( quantization_param_path, attn_tp_rank, @@ -1546,6 +1553,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin): loaded_weight, expected_fused_tp_size, deferred_scale_inv=deferred_qkv_scale_inv, + qkv_proj=self.get_submodule(name.rsplit(".", 1)[0]), ) continue @@ -1598,7 +1606,13 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin): if name in params_dict.keys(): param = params_dict[name] if "attention_sink_bias" in name: - start = get_parallel().attn_tp_rank * param.numel() + projection = unwrap_lora_layer( + self.get_submodule(name.rsplit(".", 1)[0]).qkv_proj + ) + start = ( + get_group_rank_size(projection.tp_group)[0] + * param.numel() + ) param.data.copy_( loaded_weight[start : start + param.numel()] ) @@ -1617,6 +1631,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin): deferred_qkv_scale_inv, expected_fused_tp_size, config=self.config, + model=self, ) def get_embed_and_head(self): diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index 5abeac0448..8f4dd5191d 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -20,6 +20,7 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size +from sglang.srt.distributed.utils import get_group_rank_size from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -39,6 +40,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) +from sglang.srt.lora.layers import unwrap_lora_layer from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.mimo_v2 import ( @@ -308,6 +310,7 @@ class MiMoV2MTP(MiMoV2ForCausalLM): expected_fused_tp_size=get_mimo_v2_fused_qkv_expected_tp_size( self.config ), + qkv_proj=self.get_submodule(name.rsplit(".", 1)[0]), ) continue @@ -341,7 +344,12 @@ class MiMoV2MTP(MiMoV2ForCausalLM): if name in params_dict.keys(): param = params_dict[name] if "attention_sink_bias" in name: - start = get_parallel().attn_tp_rank * param.numel() + projection = unwrap_lora_layer( + self.get_submodule(name.rsplit(".", 1)[0]).qkv_proj + ) + start = ( + get_group_rank_size(projection.tp_group)[0] * param.numel() + ) param.data.copy_(loaded_weight[start : start + param.numel()]) else: weight_loader = getattr( diff --git a/python/sglang/srt/models/opt.py b/python/sglang/srt/models/opt.py index 01794ae20e..c196f7a143 100644 --- a/python/sglang/srt/models/opt.py +++ b/python/sglang/srt/models/opt.py @@ -27,6 +27,7 @@ from sglang.srt.layers.linear import ( QKVParallelLinear, ReplicatedLinear, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput from sglang.srt.layers.pooler import Pooler, PoolingType @@ -329,6 +330,7 @@ class OPTModel(nn.Module): self.padding_idx = config.pad_token_id self.vocab_size = config.vocab_size self.pp_group = get_parallel().pp_group + self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp") self.decoder = OPTDecoder( config=config, @@ -353,8 +355,7 @@ class OPTModel(nn.Module): ) def load_kv_cache_scales(self, quantization_param_path: str) -> None: - tp_size = get_parallel().tp_size - tp_rank = get_parallel().tp_rank + tp_rank, tp_size = self._kv_cache_parallel_layout for layer_idx, scaling_factor in kv_cache_scales_loader( quantization_param_path, tp_rank, diff --git a/python/sglang/srt/models/qwen2.py b/python/sglang/srt/models/qwen2.py index 9f89bd21f9..327ec8b18e 100644 --- a/python/sglang/srt/models/qwen2.py +++ b/python/sglang/srt/models/qwen2.py @@ -40,6 +40,7 @@ from sglang.srt.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.pooler import Pooler, PoolingType @@ -342,6 +343,7 @@ class Qwen2Model(nn.Module): self.padding_idx = getattr(config, "pad_token_id", None) self.vocab_size = config.vocab_size self.pp_group = get_parallel().pp_group + self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp") if self.pp_group.is_first_rank: self.embed_tokens = VocabParallelEmbedding( @@ -454,8 +456,7 @@ class Qwen2Model(nn.Module): # factors (or else raise an exception). Thus, handled exceptions should # make sure to leave KV cache scale factors in a known good (dummy) state def load_kv_cache_scales(self, quantization_param_path: str) -> None: - tp_size = get_parallel().tp_size - tp_rank = get_parallel().tp_rank + tp_rank, tp_size = self._kv_cache_parallel_layout for layer_idx, scaling_factor in kv_cache_scales_loader( quantization_param_path, tp_rank, diff --git a/python/sglang/srt/models/solar.py b/python/sglang/srt/models/solar.py index b5d20e08ff..da0c8ba1b5 100644 --- a/python/sglang/srt/models/solar.py +++ b/python/sglang/srt/models/solar.py @@ -14,6 +14,7 @@ from sglang.srt.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, + resolve_linear_parallel_group, ) from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput from sglang.srt.layers.quantization import QuantizationConfig @@ -293,6 +294,7 @@ class SolarModel(nn.Module): self.vocab_size = config.vocab_size self.org_vocab_size = config.vocab_size self.pp_group = get_parallel().pp_group + self._kv_cache_parallel_layout = resolve_linear_parallel_group("tp") if self.pp_group.is_first_rank: self.embed_tokens = VocabParallelEmbedding( config.vocab_size, @@ -381,8 +383,7 @@ class SolarModel(nn.Module): return hidden_states def load_kv_cache_scales(self, quantization_param_path: str) -> None: - tp_size = get_parallel().tp_size - tp_rank = get_parallel().tp_rank + tp_rank, tp_size = self._kv_cache_parallel_layout for layer_idx, scaling_factor in kv_cache_scales_loader( quantization_param_path, tp_rank, diff --git a/python/sglang/srt/speculative/pp_draft_embedding.py b/python/sglang/srt/speculative/pp_draft_embedding.py index f13a512551..6f5ba5cedd 100644 --- a/python/sglang/srt/speculative/pp_draft_embedding.py +++ b/python/sglang/srt/speculative/pp_draft_embedding.py @@ -76,11 +76,21 @@ def _weight_or_none(module: Optional[nn.Module]) -> Optional[torch.Tensor]: def resolve_target_embed_and_head( - target_model: nn.Module, + target_model: nn.Module, *, draft_model: Optional[nn.Module] = None ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: """PP-safe getter: a PPMissingLayer embedding maps to ``embed=None``; any other AttributeError propagates.""" try: + if draft_model is not None and hasattr( + target_model, "get_embed_and_head_for_draft" + ): + found = _find_draft_embedding_module(draft_model, allow_replicated=True) + if found is None: + raise ValueError( + f"Draft model {draft_model.__class__.__name__} has no single " + "input embedding for target embedding sharing." + ) + return target_model.get_embed_and_head_for_draft(found[1]) return target_model.get_embed_and_head() except AttributeError: if not _target_input_embedding_is_missing(target_model): @@ -88,24 +98,38 @@ def resolve_target_embed_and_head( return None, _weight_or_none(target_model.lm_head) -def find_draft_embedding_param( - draft_model: nn.Module, -) -> Optional[Tuple[str, nn.Parameter]]: - """The draft's own input embedding: its single ``VocabParallelEmbedding``.""" +def _find_draft_embedding_module( + draft_model: nn.Module, *, allow_replicated: bool = False +) -> Optional[Tuple[str, nn.Module]]: + """Find a single vocab-parallel or named replicated input embedding.""" from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) found = [ - (f"{name}.weight", module.weight) + (f"{name}.weight", module) for name, module in draft_model.named_modules() - if isinstance(module, VocabParallelEmbedding) + if ( + isinstance(module, VocabParallelEmbedding) + or ( + allow_replicated + and isinstance(module, nn.Embedding) + and name.rsplit(".", 1)[-1] in _EMBED_ATTR_NAMES + ) + ) and not isinstance(module, ParallelLMHead) ] return found[0] if len(found) == 1 else None +def find_draft_embedding_param( + draft_model: nn.Module, +) -> Optional[Tuple[str, nn.Parameter]]: + found = _find_draft_embedding_module(draft_model) + return (found[0], found[1].weight) if found is not None else None + + def _is_input_embedding_key(key: str) -> bool: if not key.endswith(_EMBED_KEY_SUFFIXES) or _LAYER_KEY_RE.search(key): return False @@ -233,7 +257,7 @@ def resolve_draft_embed_and_head( ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: """Embed/head to bind into a draft; a stage without the target embedding loads the draft's own from the checkpoint.""" - embed, head = resolve_target_embed_and_head(target_model) + embed, head = resolve_target_embed_and_head(target_model, draft_model=draft_model) if embed is None: embed = load_draft_embedding_from_checkpoint( draft_model, model_path, revision=revision, load_config=load_config diff --git a/test/registered/unit/dllm/test_gemma4_cuda_graph.py b/test/registered/unit/dllm/test_gemma4_cuda_graph.py index 418f4de935..4198b5ae93 100644 --- a/test/registered/unit/dllm/test_gemma4_cuda_graph.py +++ b/test/registered/unit/dllm/test_gemma4_cuda_graph.py @@ -15,6 +15,7 @@ from sglang.srt.model_executor.runner.decode_cuda_graph_runner import ( ) from sglang.srt.models.gemma4_diffusion import DiffusionGemmaTextEmbedding from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, published_topology register_cpu_ci(est_time=3, suite="base-a-test-cpu") @@ -180,7 +181,7 @@ class TestGemma4GraphInputEmbeddings(unittest.TestCase): runner.load_batch(batch) -class TestGemma4VocabularyShards(unittest.TestCase): +class TestGemma4VocabularyShards(CustomTestCase): def test_padded_shards_load_and_reconstruct_soft_embeddings(self): torch.manual_seed(123) config = SimpleNamespace(vocab_size=73, hidden_size=4) @@ -192,9 +193,8 @@ class TestGemma4VocabularyShards(unittest.TestCase): with self.subTest(tp_size=tp_size): partials = [] for rank in range(tp_size): - with patch( - "sglang.srt.layers.vocab_parallel_embedding.get_parallel", - return_value=SimpleNamespace(tp_rank=rank, tp_size=tp_size), + with published_topology( + device="cpu", tp_size=tp_size, ranks={"world_rank": rank} ): embedding = DiffusionGemmaTextEmbedding(config) embedding.weight_loader(embedding.weight, weight) diff --git a/test/registered/unit/models/test_attention_sink_loader_layout.py b/test/registered/unit/models/test_attention_sink_loader_layout.py new file mode 100644 index 0000000000..7016e3e0ff --- /dev/null +++ b/test/registered/unit/models/test_attention_sink_loader_layout.py @@ -0,0 +1,210 @@ +"""Attention sink checkpoints follow the constructed head partitions.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.lora.layers import BaseLayerWithLoRA, unwrap_lora_layer +from sglang.srt.models import gpt_oss +from sglang.srt.models.gpt_oss import GptOssAttention, GptOssForCausalLM +from sglang.srt.models.mimo_v2 import MiMoV2Attention, MiMoV2ForCausalLM +from sglang.srt.models.mimo_v2_nextn import MiMoV2MTP +from sglang.srt.runtime_context import SpawnRanks, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.parallel_groups import parallel_scope, publish, rank_size +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=15, stage="nightly", runner_config="1-gpu-large") + + +def loading_scope(changed): + if not changed: + return nullcontext() + return parallel_scope( + tp_size=1, + tp_rank=0, + tp_group=None, + attn_tp_size=1, + attn_tp_rank=0, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + moe_ep_group=None, + moe_dp_size=1, + moe_dp_rank=0, + ) + + +def build_model(kind, dtype=torch.bfloat16, kv_heads=1, device="cuda"): + model_type = { + "gpt": GptOssForCausalLM, + "mimo": MiMoV2ForCausalLM, + "mtp": MiMoV2MTP, + }[kind] + model = model_type.__new__(model_type) + torch.nn.Module.__init__(model) + model.config = SimpleNamespace( + num_hidden_layers=1, + n_routed_experts=0, + tie_word_embeddings=False, + encoder_only=False, + ) + model.quant_config = None + model._is_multimodal = False + with torch.device(device): + if kind == "gpt": + attention = GptOssAttention( + 64, + 8, + kv_heads, + head_dim=8, + max_position_embeddings=16, + layer_type="full_attention", + params_dtype=dtype, + ) + else: + attention = MiMoV2Attention( + 64, + 8, + kv_heads, + head_dim=8, + v_head_dim=16, + max_position_embeddings=16, + attention_sink_bias=True, + ) + model.model = torch.nn.Module() + if kind == "mtp": + model.model.mtp_block = torch.nn.Module() + model.model.mtp_block.self_attn = attention + name = "model.mtp.layers.0.self_attn.attention_sink_bias" + else: + model.model.layers = torch.nn.ModuleList([torch.nn.Module()]) + model.model.layers[0].self_attn = attention + name = "model.layers.0.self_attn." + ( + "sinks" if kind == "gpt" else "attention_sink_bias" + ) + return model, attention, name + + +def load_sinks( + model, + attention, + name, + *, + changed=False, + offset=0, + source_heads=8, + cpu_padding=False, +): + param = ( + attention.sinks + if isinstance(attention, GptOssAttention) + else attention.attention_sink_bias + ) + source = ( + torch.arange(source_heads, device=param.device, dtype=torch.float32) + offset + ) / 16 + start = rank_size(unwrap_lora_layer(attention.qkv_proj))[0] * param.numel() + expected = source[start : start + param.numel()].to(param.dtype) + if cpu_padding and expected.numel() < param.numel(): + expected = torch.cat( + (expected, expected.new_zeros(param.numel() - expected.numel())) + ) + with loading_scope(changed), patch.object(gpt_oss, "_is_cpu", cpu_padding): + if isinstance(model, GptOssForCausalLM): + model._load_normal_weights( + [(name, source)], is_nextn=False, weight_name_mapping=None + ) + else: + model.load_weights([(name, source)]) + torch.testing.assert_close(param, expected, rtol=0, atol=0) + return param + + +@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU") +class TestAttentionSinkLoaderLayout(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + original = torch.get_default_dtype() + self.addCleanup(torch.set_default_dtype, original) + + def check_loads(self, changed, wrapped=False): + for dp in (1, 2): + for rank in range(4): + for dtype in (torch.float32, torch.bfloat16): + reset_context() + publish( + ServerArgs( + model_path="dummy", + device="cuda", + tp_size=4, + attn_dp_size=dp, + attention_backend="trtllm_mha" + if dtype == torch.float32 + else "flashinfer", + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + torch.set_default_dtype(dtype) + for kind in ("gpt", "mimo", "mtp"): + for kv_heads in (1, 4): + model, attention, name = build_model(kind, dtype, kv_heads) + if wrapped: + attention.qkv_proj = BaseLayerWithLoRA( + attention.qkv_proj, Mock() + ) + for offset in (0, 11): + load_sinks( + model, + attention, + name, + changed=changed, + offset=offset, + ) + + def test_native_model_loaders_after_scope_exit(self): + self.check_loads(True) + + def test_wrapped_attention_sinks_after_scope_exit(self): + self.check_loads(True, wrapped=True) + + def test_gpt_cpu_checkpoint_tail_padding(self): + # Exercise the CPU checkpoint branch with real CPU parameters/tensors. + # This tests loading, without requiring a CPU attention kernel. + for dp in (1, 2): + for rank in range(4): + reset_context() + publish( + ServerArgs( + model_path="dummy", device="cuda", tp_size=4, attn_dp_size=dp + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + model, attention, name = build_model("gpt", device="cpu") + for changed in (False, True): + for source_heads in (7, 8, 9): + load_sinks( + model, + attention, + name, + changed=changed, + source_heads=source_heads, + cpu_padding=True, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_dots_fused_gate_loader_layout.py b/test/registered/unit/models/test_dots_fused_gate_loader_layout.py new file mode 100644 index 0000000000..8905d93565 --- /dev/null +++ b/test/registered/unit/models/test_dots_fused_gate_loader_layout.py @@ -0,0 +1,272 @@ +"""Mixed replicated Q/KV and sharded gate checkpoints follow attention layout.""" + +import unittest +from contextlib import nullcontext +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.configs.dots3 import Dots3Config +from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod +from sglang.srt.lora.layers import BaseLayerWithLoRA, unwrap_lora_layer +from sglang.srt.models.dots3_common.modeling import ( + Dots3AttentionMLA, + Dots3LanguageModelForCausalLM, +) +from sglang.srt.runtime_context import SpawnRanks, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.parallel_groups import parallel_scope, publish, rank_size +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large") + + +def loading_scope(changed): + if not changed: + return nullcontext() + return parallel_scope( + tp_size=1, + tp_rank=0, + tp_group=None, + attn_tp_size=1, + attn_tp_rank=0, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + moe_ep_group=None, + moe_dp_size=1, + moe_dp_rank=0, + ) + + +def build_model( + gate_type="headwise", layer_type="full_attention", fp8=False, nextn=False +): + config = Dots3Config( + hidden_size=128, + intermediate_size=256, + num_hidden_layers=1, + max_position_embeddings=32, + layer_types=[layer_type], + attention_gate_type=gate_type, + q_lora_rank=128, + kv_lora_rank=128, + qk_nope_head_dim=128, + qk_rope_head_dim=64, + num_attention_heads=8, + num_key_value_heads=8, + v_head_dim=128, + swa_attention_gate_type=gate_type, + swa_q_lora_rank=128, + swa_kv_lora_rank=128, + swa_qk_nope_head_dim=128, + swa_qk_rope_head_dim=64, + swa_num_attention_heads=8, + swa_num_key_value_heads=8, + swa_v_head_dim=128, + n_routed_experts=0, + n_shared_experts=0, + num_nextn_predict_layers=1 if nextn else 0, + ) + quant = ( + Fp8Config(is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128]) + if fp8 + else None + ) + model = Dots3LanguageModelForCausalLM.__new__(Dots3LanguageModelForCausalLM) + torch.nn.Module.__init__(model) + model.config = config + model.quant_config = quant + model.num_fused_shared_experts = 0 + with torch.device("cuda"): + attention = Dots3AttentionMLA(config, quant, layer_id=0) + model.model = torch.nn.Module() + model.model.start_layer = 0 + model.model.end_layer = 1 + if nextn: + model.model.heads = torch.nn.ModuleList([torch.nn.Module()]) + model.model.heads[0].decoder = torch.nn.Module() + model.model.heads[0].decoder.self_attn = attention + else: + model.model.layers = torch.nn.ModuleList([torch.nn.Module()]) + model.model.layers[0].self_attn = attention + if fp8: + assert isinstance( + attention.fused_qkv_a_g_proj_with_mqa.quant_method, Fp8LinearMethod + ) + return model, attention + + +def checkpoint_parts(attention, offset): + module = attention.fused_qkv_a_g_proj_with_mqa + widths = ( + attention.q_lora_rank, + attention.kv_lora_rank + attention.qk_rope_head_dim, + attention.num_heads + * (1 if attention.attention_gate_type == "headwise" else attention.v_head_dim), + ) + parts = {} + for index, (name, width) in enumerate( + zip(("q_a_proj", "kv_a_proj_with_mqa", "g_proj"), widths) + ): + weight = ( + ( + ( + torch.arange(width * 128, device=module.weight.device) + + offset + + index * 11 + ) + % 31 + - 15 + ) + .reshape(width, 128) + .to(module.weight.dtype) + ) + parts[name + ".weight"] = weight + if hasattr(module, "weight_scale_inv"): + parts[name + ".weight_scale_inv"] = ( + ( + torch.arange((width + 127) // 128, device=weight.device) + + offset + + index + ) + % 3 + + 1 + ).float().reshape(-1, 1) / 128 + return parts + + +def load_fused( + model, attention, *, changed=False, offset=0, reverse=False, nextn=False +): + module = attention.fused_qkv_a_g_proj_with_mqa + parts = checkpoint_parts(attention, offset) + names = list(parts) + if reverse: + names.reverse() + weights = [("model.layers.0.self_attn." + name, parts[name]) for name in names] + # Keep this fixture focused on checkpoint fusion. The separate model hook + # binds kv_b views and requantizes unrelated attention/MLP parameters. + with loading_scope(changed), patch.object(model, "post_load_weights") as post: + model.load_weights(weights, is_nextn=nextn) + post.assert_called_once() + owner = unwrap_lora_layer(attention.q_b_proj) + gate = parts["g_proj.weight"].chunk(rank_size(owner)[1], 0)[rank_size(owner)[0]] + + def pad(t, target): + return torch.cat((t, t.new_zeros(target - t.shape[0], t.shape[1])), 0) + + expected = torch.cat( + ( + parts["q_a_proj.weight"], + pad( + parts["kv_a_proj_with_mqa.weight"], + attention.kv_lora_rank + attention.qk_rope_head_dim_padded, + ), + pad(gate, attention.g_proj_local_dim_padded), + ), + 0, + ) + if module.weight.dtype == torch.float8_e4m3fn: + torch.testing.assert_close( + module.weight.view(torch.uint8), expected.view(torch.uint8), rtol=0, atol=0 + ) + start = rank_size(owner)[0] * gate.shape[0] // 128 + end = ((rank_size(owner)[0] + 1) * gate.shape[0] + 127) // 128 + expected_scale = torch.cat( + ( + parts["q_a_proj.weight_scale_inv"], + parts["kv_a_proj_with_mqa.weight_scale_inv"], + parts["g_proj.weight_scale_inv"][start:end], + ), + 0, + ) + torch.testing.assert_close( + module.weight_scale_inv, expected_scale, rtol=0, atol=0 + ) + else: + torch.testing.assert_close(module.weight, expected, rtol=0, atol=0) + return module + + +@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU") +class TestDotsFusedGateLoaderLayout(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + original = torch.get_default_dtype() + self.addCleanup(torch.set_default_dtype, original) + torch.set_default_dtype(torch.bfloat16) + + def check_loads(self, changed, nextn=False, wrapped=False): + for dp in (1, 2): + for rank in range(4): + reset_context() + publish( + ServerArgs( + model_path="dummy", device="cuda", tp_size=4, attn_dp_size=dp + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + for gate in ("headwise", "elementwise"): + for layer in ("full_attention", "sliding_attention"): + for fp8 in (False, True): + model, attention = build_model(gate, layer, fp8, nextn) + if wrapped: + attention.q_b_proj = BaseLayerWithLoRA( + attention.q_b_proj, Mock() + ) + for offset in (0, 11): + for reverse in (False, True): + with self.subTest( + dp=dp, + rank=rank, + gate=gate, + layer=layer, + fp8=fp8, + offset=offset, + reverse=reverse, + ): + load_fused( + model, + attention, + changed=changed, + offset=offset, + reverse=reverse, + nextn=nextn, + ) + + def test_native_fused_parameters_after_scope_exit(self): + self.check_loads(True) + + def test_mtp_checkpoint_mapping_after_scope_exit(self): + self.check_loads(True, nextn=True) + + def test_wrapped_gate_owner_after_scope_exit(self): + self.check_loads(True, wrapped=True) + + def test_gate_keeps_rank_when_width_is_unchanged(self): + publish( + ServerArgs(model_path="dummy", device="cuda", tp_size=4, attn_dp_size=2), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + for gate in ("headwise", "elementwise"): + for fp8 in (False, True): + model, attention = build_model(gate, fp8=fp8) + with parallel_scope( + tp_rank=0, attn_tp_rank=0, attn_dp_rank=0, moe_tp_rank=0 + ): + load_fused(model, attention) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_gemma_shared_vocab_layout.py b/test/registered/unit/models/test_gemma_shared_vocab_layout.py new file mode 100644 index 0000000000..c0b880b6a2 --- /dev/null +++ b/test/registered/unit/models/test_gemma_shared_vocab_layout.py @@ -0,0 +1,304 @@ +"""Gemma embedding exports retain their source or recipient layout.""" + +import copy +import socket +import unittest +from contextlib import nullcontext + +import torch +from transformers import ( + Gemma3Config, + Gemma3TextConfig, + Gemma4AssistantConfig, + Gemma4Config, + Gemma4TextConfig, + Gemma4VisionConfig, + LlamaConfig, + SiglipVisionConfig, +) + +from sglang.srt.configs.load_config import LoadConfig, LoadFormat +from sglang.srt.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.layers.dp_attention import initialize_dp_attention +from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding +from sglang.srt.models.gemma3_causal import Gemma3ForCausalLM +from sglang.srt.models.gemma3_mm import Gemma3ForConditionalGeneration +from sglang.srt.models.gemma4_causal import Gemma4ForCausalLM +from sglang.srt.models.gemma4_mm import Gemma4ForConditionalGeneration +from sglang.srt.models.gemma4_mtp import Gemma4AssistantForCausalLM +from sglang.srt.models.llama_eagle3 import LlamaForCausalLMEagle3 +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.srt.speculative.pp_draft_embedding import ( + find_draft_embedding_param, + resolve_draft_embed_and_head, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=18, stage="nightly", runner_config="1-gpu-large") + + +def build_source(kind, *, tied=True, vocab=512): + extra = {} + if kind in ("gemma4", "gemma4_mm"): + extra.update( + global_head_dim=16, + vocab_size_per_layer_input=vocab, + hidden_size_per_layer_input=8, + ) + cls = Gemma4TextConfig if kind in ("gemma4", "gemma4_mm") else Gemma3TextConfig + cfg = cls( + hidden_size=128, + intermediate_size=256, + num_hidden_layers=2, + num_attention_heads=8, + num_key_value_heads=4, + head_dim=16, + vocab_size=vocab, + max_position_embeddings=32, + sliding_window=16, + layer_types=["sliding_attention", "full_attention"], + tie_word_embeddings=tied, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + **extra, + ) + if kind == "gemma3_mm": + vision = SiglipVisionConfig( + hidden_size=64, + intermediate_size=128, + num_hidden_layers=1, + num_attention_heads=4, + image_size=32, + patch_size=16, + ) + cfg = Gemma3Config( + text_config=cfg.to_dict(), + vision_config=vision.to_dict(), + mm_tokens_per_image=4, + ) + if kind == "gemma4_mm": + cfg = Gemma4Config( + text_config=cfg, + vision_config=Gemma4VisionConfig( + hidden_size=64, + intermediate_size=128, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=4, + head_dim=16, + patch_size=16, + position_embedding_size=32, + ), + audio_config=None, + ) + model_cls = { + "gemma3": Gemma3ForCausalLM, + "gemma4": Gemma4ForCausalLM, + "gemma3_mm": Gemma3ForConditionalGeneration, + "gemma4_mm": Gemma4ForConditionalGeneration, + }[kind] + with torch.device("cuda"): + return model_cls(cfg) + + +def build_draft(vocab=512, *, embedding_only=False): + if embedding_only: + draft = torch.nn.Module() + draft.model = torch.nn.Module() + with torch.device("cuda"): + draft.model.embed_tokens = VocabParallelEmbedding(vocab, 128) + return draft + config = LlamaConfig( + hidden_size=128, + intermediate_size=256, + num_hidden_layers=1, + num_attention_heads=8, + num_key_value_heads=4, + head_dim=16, + vocab_size=vocab, + max_position_embeddings=32, + tie_word_embeddings=True, + draft_vocab_size=vocab, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + ) + with torch.device("cuda"): + return LlamaForCausalLMEagle3(config) + + +def source_weights(model): + if isinstance(model, Gemma4ForConditionalGeneration): + embed = model.language_model.embed_tokens.weight + return embed, embed + source = getattr(model, "language_model", model) + return source.model.embed_tokens.weight, source.lm_head.weight + + +def fill_source(model, version=0): + embed, head = source_weights(model) + for index, weight in enumerate((embed, head)): + if index == 1 and weight is embed: + continue + values = ( + ( + torch.arange(weight.numel(), device=weight.device).reshape(weight.shape) + % 127 + ) + - 63 + + version + + index + ) / 64 + with torch.no_grad(): + weight.copy_(values) + return embed, head + + +def share(model, draft): + return resolve_draft_embed_and_head( + target_model=model, + draft_model=draft, + model_path="/nonexistent", + revision=None, + load_config=LoadConfig(load_format=LoadFormat.DUMMY), + ) + + +def changed_scope(changed, *, size=4, rank=3): + if not changed: + return nullcontext() + return get_parallel().override( + tp_size=size, + tp_rank=rank, + tp_group=None, + attn_tp_size=size, + attn_tp_rank=rank, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=size, + moe_tp_rank=rank, + moe_ep_size=1, + moe_ep_rank=0, + moe_ep_group=None, + moe_dp_size=1, + moe_dp_rank=0, + ) + + +@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU") +class TestGemmaSharedVocabLayout(CustomTestCase): + def setUp(self): + if torch.distributed.is_initialized(): + self.skipTest("requires an isolated distributed test process") + reset_context() + self.addCleanup(reset_context) + original = torch.get_default_dtype() + self.addCleanup(torch.set_default_dtype, original) + torch.set_default_dtype(torch.bfloat16) + server = ServerArgs(model_path="dummy", device="cuda", tp_size=1) + publish(server, role="test", ranks=SpawnRanks(world_rank=0, gpu_id=0)) + torch.cuda.set_device(0) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + init_distributed_environment( + world_size=1, + rank=0, + local_rank=0, + distributed_init_method=f"tcp://127.0.0.1:{port}", + ) + self.addCleanup(destroy_distributed_environment) + initialize_model_parallel() + self.addCleanup(destroy_model_parallel) + initialize_dp_attention(server) + + def test_sharing_uses_constructed_draft_after_scope_exit(self): + for kind in ("gemma3", "gemma4", "gemma3_mm"): + model = build_source(kind) + expected = fill_source(model) + draft = build_draft() + with changed_scope(True): + actual = share(model, draft) + for result, weight in zip(actual, expected): + self.assertIs(result, weight) + + def test_draft_rank_and_width_override_source_export(self): + for kind in ("gemma3", "gemma4", "gemma3_mm"): + model = build_source(kind) + embed, head = fill_source(model) + with changed_scope(True): + draft = build_draft(embedding_only=True) + actual = share(model, draft) + for result, weight in zip(actual, (embed, head)): + torch.testing.assert_close(result, weight[384:512], rtol=0, atol=0) + self.assertEqual( + result.untyped_storage().data_ptr(), + weight.untyped_storage().data_ptr(), + ) + self.assertIs( + find_draft_embedding_param(draft)[1], draft.model.embed_tokens.weight + ) + + def test_native_gemma4_mtp_keeps_full_target_embedding(self): + """Replicated MTP recipients keep the full embedding after scope exit.""" + for kind in ("gemma4", "gemma4_mm"): + for tied in (True, False): + with self.subTest(kind=kind, tied=tied): + model = build_source(kind, tied=tied, vocab=513) + embed, head = fill_source(model) + text_config = ( + model.config.text_config + if kind == "gemma4_mm" + else model.config + ) + assistant_text = copy.deepcopy(text_config) + assistant_text.hidden_size_per_layer_input = 0 + assistant_text.vocab_size_per_layer_input = 0 + assistant_text.enable_moe_block = False + assistant_text.use_double_wide_mlp = False + assistant_text.num_kv_shared_layers = ( + assistant_text.num_hidden_layers + ) + config = Gemma4AssistantConfig( + text_config=assistant_text, + backbone_hidden_size=text_config.hidden_size, + ) + with changed_scope(True), torch.device("cuda"): + draft = Gemma4AssistantForCausalLM(config) + actual_embed, actual_head = share(model, draft) + self.assertIs(actual_embed, embed) + self.assertIs(actual_head, head) + draft.set_embed_and_head(actual_embed, actual_head) + ids = torch.tensor([0, 512], device="cuda") + torch.testing.assert_close( + torch.nn.functional.embedding(ids, draft.target_embed_weight), + embed[ids], + rtol=0, + atol=0, + ) + self.assertIsNone(find_draft_embedding_param(draft)) + + def test_shared_views_see_source_weight_updates(self): + model = build_source("gemma3") + fill_source(model) + with changed_scope(True): + draft = build_draft(embedding_only=True) + embed, head = share(model, draft) + full_embed, full_head = fill_source(model, version=13) + torch.testing.assert_close(embed, full_embed[384:512], rtol=0, atol=0) + torch.testing.assert_close(head, full_head[384:512], rtol=0, atol=0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_interns1pro_fope_loader_layout.py b/test/registered/unit/models/test_interns1pro_fope_loader_layout.py new file mode 100644 index 0000000000..d71520268f --- /dev/null +++ b/test/registered/unit/models/test_interns1pro_fope_loader_layout.py @@ -0,0 +1,197 @@ +"""FOPE coefficient loading follows the destination attention partition.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock + +import torch + +from sglang.srt.lora.layers import BaseLayerWithLoRA, unwrap_lora_layer +from sglang.srt.models.interns1pro import ( + InternS1ProForConditionalGeneration, + InternS1ProTextAttention, +) +from sglang.srt.runtime_context import SpawnRanks, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.parallel_groups import parallel_scope, publish, rank_size +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large") + + +def loading_scope(changed): + if not changed: + return nullcontext() + return parallel_scope( + tp_size=1, + tp_rank=0, + tp_group=None, + attn_tp_size=1, + attn_tp_rank=0, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + moe_ep_group=None, + moe_dp_size=1, + moe_dp_rank=0, + ) + + +def build_model(kv_heads, head_dim=16): + model = InternS1ProForConditionalGeneration.__new__( + InternS1ProForConditionalGeneration + ) + torch.nn.Module.__init__(model) + model.config = SimpleNamespace(num_experts=0) + with torch.device("cuda"): + attention = InternS1ProTextAttention( + 128, + 8, + kv_heads, + head_dim=head_dim, + max_position_embeddings=32, + rope_scaling={ + "rope_type": "default", + "fope_sep_head": True, + "num_inv_freq": 3, + }, + config=SimpleNamespace(), + ) + model.model = torch.nn.Module() + model.model.layers = torch.nn.ModuleList([torch.nn.Module(), torch.nn.Module()]) + model.model.layers[0].self_attn = attention + with torch.device("cuda"): + second = InternS1ProTextAttention( + 128, + 8, + kv_heads, + layer_id=1, + head_dim=head_dim, + max_position_embeddings=32, + rope_scaling={ + "rope_type": "default", + "fope_sep_head": True, + "num_inv_freq": 3, + }, + config=SimpleNamespace(), + ) + model.model.layers[1].self_attn = second + assert attention.rotary_emb is second.rotary_emb + return model, attention + + +def load_coefficients(model, attention, *, changed=False, offset=0, top_level=False): + weights = [] + for index, coefficient in enumerate(("cos_coef", "sin_coef")): + param = getattr(attention.rotary_emb, coefficient) + shape = (attention.total_num_kv_heads, *param.shape[1:]) + full = ( + ( + torch.arange(shape[0] * shape[1] * shape[2], device=param.device) + + offset + + index * 11 + ) + % 37 + - 18 + ).float().reshape(shape) / 64 + owner = unwrap_lora_layer(attention.qkv_proj) + first = ( + rank_size(owner)[0] + // max(1, rank_size(owner)[1] // shape[0]) + * param.shape[0] + ) + expected = full[first : first + param.shape[0]] + name = "model.rotary_emb." + coefficient + if top_level: + weights.append(("model.language_model.rotary_emb." + coefficient, full)) + else: + with loading_scope(changed): + model._load_fope_weights(name, full, dict(model.named_parameters())) + torch.testing.assert_close(param, expected, rtol=0, atol=0) + weights.append((param, expected)) + if top_level: + sources = [item for item in weights if isinstance(item[0], str)] + with loading_scope(changed): + model.load_weights(sources) + for param, expected in weights: + if not isinstance(param, str): + torch.testing.assert_close(param, expected, rtol=0, atol=0) + return attention.rotary_emb + + +@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU") +class TestInternS1ProFopeLoaderLayout(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + original = torch.get_default_dtype() + self.addCleanup(torch.set_default_dtype, original) + torch.set_default_dtype(torch.bfloat16) + + def check_loads(self, changed, top_level=False, wrapped=False): + for dp in (1, 2): + for rank in range(4): + reset_context() + publish( + ServerArgs( + model_path="dummy", device="cuda", tp_size=4, attn_dp_size=dp + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + for kv_heads in (1, 2, 8): + for head_dim in (8, 16): + model, attention = build_model(kv_heads, head_dim) + if wrapped: + attention.qkv_proj = BaseLayerWithLoRA( + attention.qkv_proj, Mock() + ) + for offset in (0, 13): + with self.subTest( + dp=dp, + rank=rank, + kv_heads=kv_heads, + head_dim=head_dim, + offset=offset, + ): + load_coefficients( + model, + attention, + changed=changed, + offset=offset, + top_level=top_level, + ) + + def test_native_attention_coefficients_after_scope_exit(self): + self.check_loads(True) + + def test_checkpoint_name_mapping_after_scope_exit(self): + self.check_loads(True, top_level=True) + + def test_wrapped_projection_coefficients_after_scope_exit(self): + self.check_loads(True, top_level=True, wrapped=True) + + def test_coefficient_shards_keep_rank_when_width_is_unchanged(self): + publish( + ServerArgs(model_path="dummy", device="cuda", tp_size=4, attn_dp_size=2), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + for kv_heads in (1, 2, 8): + model, attention = build_model(kv_heads) + with parallel_scope( + tp_rank=0, attn_tp_rank=0, attn_dp_rank=0, moe_tp_rank=0 + ): + load_coefficients(model, attention) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_kv_calibration_loader_layout.py b/test/registered/unit/models/test_kv_calibration_loader_layout.py new file mode 100644 index 0000000000..d490c80f4a --- /dev/null +++ b/test/registered/unit/models/test_kv_calibration_loader_layout.py @@ -0,0 +1,261 @@ +"""External KV calibration metadata follows the model's construction layout.""" + +import json +import socket +import tempfile +import unittest +from contextlib import nullcontext +from pathlib import Path +from unittest.mock import patch + +import torch +from transformers import ApertusConfig, Glm4Config, LlamaConfig, OPTConfig, Qwen2Config + +from sglang.srt.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.layers.dp_attention import initialize_dp_attention +from sglang.srt.models import hunyuan +from sglang.srt.models.apertus import ApertusModel +from sglang.srt.models.arcee import ArceeModel +from sglang.srt.models.glm4 import Glm4Model +from sglang.srt.models.hunyuan import HunYuanMoEV1ForCausalLM +from sglang.srt.models.llama import LlamaModel +from sglang.srt.models.mimo_v2 import MiMoV2Model +from sglang.srt.models.opt import OPTModel +from sglang.srt.models.qwen2 import Qwen2Model +from sglang.srt.models.solar import SolarModel +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large") + +MODELS = { + "llama": (LlamaModel, LlamaConfig), + "qwen2": (Qwen2Model, Qwen2Config), + "glm4": (Glm4Model, Glm4Config), + "opt": (OPTModel, OPTConfig), + "apertus": (ApertusModel, ApertusConfig), + "arcee": (ArceeModel, LlamaConfig), + "solar": (SolarModel, LlamaConfig), + "hunyuan": (HunYuanMoEV1ForCausalLM, LlamaConfig), + "mimo": (MiMoV2Model, LlamaConfig), +} + + +def build_model(kind): + cls, cfg = MODELS[kind] + extra = {} + if kind == "apertus": + extra.update( + hidden_act="xielu", + rope_parameters={"rope_type": "default", "rope_theta": 10000.0}, + ) + if kind == "arcee": + extra.update(hidden_act="relu2") + if kind == "solar": + extra.update(bskcn_1=[0], bskcn_2=[], bskcn_3=[1], bskcn_4=[]) + if kind == "hunyuan": + extra.update(attention_head_dim=16, num_experts=1) + if kind == "mimo": + extra.update( + layernorm_epsilon=1e-6, + hybrid_layer_pattern=[0, 1], + swa_num_attention_heads=8, + swa_num_key_value_heads=4, + swa_head_dim=16, + sliding_window_size=16, + moe_layer_freq=[0, 0], + ) + config = cfg( + hidden_size=128, + intermediate_size=256, + ffn_dim=256, + num_hidden_layers=2, + num_attention_heads=8, + num_key_value_heads=4, + head_dim=16, + vocab_size=256, + max_position_embeddings=32, + word_embed_proj_dim=128, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + **extra, + ) + with torch.device("cuda"): + return cls(config) + + +def attention_layers(model): + if isinstance(model, OPTModel): + layers = model.decoder.layers + elif isinstance(model, HunYuanMoEV1ForCausalLM): + layers = model.model.layers + else: + layers = model.layers + return [layer.self_attn.attn for layer in layers] + + +def calibration_data(model, tp_size, version): + return dict( + model_type=model.config.model_type, + kv_cache=dict( + dtype="float8_e4m3fn", + scaling_factor={ + rank: { + layer: (rank * 7 + layer * 3 + version + 1) / 32 + for layer in range(model.config.num_hidden_layers) + } + for rank in range(tp_size) + }, + ), + ) + + +def loading_scope(changed): + if not changed: + return nullcontext() + # Width4/rank3 is valid during loading even when CI constructed at TP1. + # No model construction or kernel executes within this temporary scope. + return get_parallel().override( + tp_size=4, + tp_rank=3, + tp_group=None, + attn_tp_size=4, + attn_tp_rank=3, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=4, + moe_tp_rank=3, + moe_ep_size=1, + moe_ep_rank=0, + moe_ep_group=None, + moe_dp_size=1, + moe_dp_rank=0, + ) + + +def load_scales(model, directory, *, tp_rank, tp_size, version=0, changed=False): + data = calibration_data(model, tp_size, version) + path = Path(directory) / "calibration.json" + path.write_text(json.dumps(data)) + if isinstance(model, HunYuanMoEV1ForCausalLM): + # This legacy native endpoint parses the file, then rejects the missing + # kv_scale attribute. Verify its real selection and preserve that guard. + selected = [] + original = hunyuan.kv_cache_scales_loader + + def record(*args, **kwargs): + rows = list(original(*args, **kwargs)) + selected.extend(rows) + return rows + + with ( + patch.object(hunyuan, "kv_cache_scales_loader", record), + loading_scope(changed), + ): + try: + model.load_kv_cache_scales(str(path)) + except RuntimeError as error: + assert "KV cache scaling factor attribute" in str(error) + else: + raise AssertionError("Expected the existing HunYuan kv_scale guard") + assert selected == list(data["kv_cache"]["scaling_factor"][tp_rank].items()) + return [(layer.k_scale, layer.v_scale) for layer in attention_layers(model)] + with loading_scope(changed): + model.load_kv_cache_scales(str(path)) + result = [(layer.k_scale, layer.v_scale) for layer in attention_layers(model)] + expected = [ + (data["kv_cache"]["scaling_factor"][tp_rank][i],) * 2 + for i in range(model.config.num_hidden_layers) + ] + assert result == expected, (type(model).__name__, result, expected) + return result + + +@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU") +class TestKvCalibrationLoaderLayout(CustomTestCase): + def setUp(self): + if torch.distributed.is_initialized(): + self.skipTest("requires an isolated distributed test process") + reset_context() + self.addCleanup(reset_context) + original = torch.get_default_dtype() + self.addCleanup(torch.set_default_dtype, original) + torch.set_default_dtype(torch.bfloat16) + server = ServerArgs(model_path="dummy", device="cuda", tp_size=1) + publish(server, role="test", ranks=SpawnRanks(world_rank=0, gpu_id=0)) + torch.cuda.set_device(0) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + init_distributed_environment( + world_size=1, + rank=0, + local_rank=0, + distributed_init_method=f"tcp://127.0.0.1:{port}", + ) + self.addCleanup(destroy_distributed_environment) + initialize_model_parallel() + self.addCleanup(destroy_model_parallel) + initialize_dp_attention(server) + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.directory = tmp.name + + def check_loads(self, changed): + for kind in MODELS: + model = build_model(kind) + for version in (0, 13): + with self.subTest(kind=kind, version=version): + load_scales( + model, + self.directory, + tp_rank=0, + tp_size=1, + version=version, + changed=changed, + ) + + def test_real_model_calibration_after_scope_exit(self): + self.check_loads(True) + + def test_existing_invalid_file_fallback_keeps_loaded_scales(self): + for kind in MODELS: + model = build_model(kind) + expected = load_scales(model, self.directory, tp_rank=0, tp_size=1) + for changed in (False, True): + for error in ("missing", "json", "model_type", "dtype", "ranks"): + path = Path(self.directory) / "invalid.json" + data = calibration_data(model, 1, 0) + if error == "missing": + path = Path(self.directory) / "missing.json" + elif error == "json": + path.write_text("{") + else: + if error == "model_type": + data["model_type"] = "wrong_model_type" + if error == "dtype": + data["kv_cache"]["dtype"] = "float32" + if error == "ranks": + data["kv_cache"]["scaling_factor"] = {} + path.write_text(json.dumps(data)) + with loading_scope(changed): + model.load_kv_cache_scales(str(path)) + self.assertEqual( + [(a.k_scale, a.v_scale) for a in attention_layers(model)], + expected, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_mimo_fused_qkv_loader_layout.py b/test/registered/unit/models/test_mimo_fused_qkv_loader_layout.py new file mode 100644 index 0000000000..ce25878a43 --- /dev/null +++ b/test/registered/unit/models/test_mimo_fused_qkv_loader_layout.py @@ -0,0 +1,303 @@ +"""MiMo fused QKV checkpoint preprocessing follows the destination projection.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock + +import torch + +from sglang.srt.layers.linear import QKVParallelLinear +from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod +from sglang.srt.lora.layers import BaseLayerWithLoRA, unwrap_lora_layer +from sglang.srt.models.mimo_v2 import ( + MiMoV2ForCausalLM, + _resolve_deferred_qkv_scale_inv, + load_mimo_v2_qkv_proj_weight, +) +from sglang.srt.models.mimo_v2_nextn import MiMoV2MTP +from sglang.srt.runtime_context import SpawnRanks, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.parallel_groups import parallel_scope, publish, rank_size +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large") + +WEIGHT_NAME = "model.layers.0.self_attn.qkv_proj.weight" +SCALE_NAME = WEIGHT_NAME.replace(".weight", ".weight_scale_inv") + + +def loading_scope(changed): + if not changed: + return nullcontext() + return parallel_scope( + tp_size=1, + tp_rank=0, + tp_group=None, + attn_tp_size=1, + attn_tp_rank=0, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + moe_ep_group=None, + moe_dp_size=1, + moe_dp_rank=0, + ) + + +def build_projection(dtype=torch.bfloat16, fp8=False): + config = ( + Fp8Config(is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128]) + if fp8 + else None + ) + with torch.device("cuda"): + projection = QKVParallelLinear( + 128, + 32, + 64, + 32, + v_head_size=16, + bias=False, + params_dtype=dtype, + quant_config=config, + parallel_group="attn_tp", + ) + if fp8: + assert isinstance(projection.quant_method, Fp8LinearMethod) + model = torch.nn.Module() + model.model = torch.nn.Module() + model.model.layers = torch.nn.ModuleList([torch.nn.Module()]) + model.model.layers[0].self_attn = torch.nn.Module() + model.model.layers[0].self_attn.qkv_proj = projection + return model, projection + + +def values(shape, dtype, offset=0): + count = 1 + for size in shape: + count *= size + return ( + ((torch.arange(count, device="cuda") + offset) % 31 - 15) + .reshape(shape) + .to(dtype) + ) + + +def load_fused(projection, *, changed=False, ckpt_tp=None, sharded=False, offset=0): + destination = projection + projection = unwrap_lora_layer(projection) + weight = projection.weight + full = values( + (weight.shape[0] * rank_size(projection)[1], weight.shape[1]), + weight.dtype, + offset, + ) + shards = full.chunk(rank_size(projection)[1], dim=0) + data = shards[rank_size(projection)[0]] if sharded else full + with loading_scope(changed): + load_mimo_v2_qkv_proj_weight( + WEIGHT_NAME, weight, data, ckpt_tp, qkv_proj=destination + ) + torch.testing.assert_close(weight, shards[rank_size(projection)[0]], rtol=0, atol=0) + return weight + + +def load_deferred(model, projection, *, changed=False, offset=0): + destination = projection + projection = unwrap_lora_layer(projection) + ckpt_tp = 8 + rows = projection.weight.shape[0] * rank_size(projection)[1] + full = values((rows, 128), torch.float8_e4m3fn, offset) + # Each checkpoint shard has 448 rows, requiring four 128-row scales. + scale = ((torch.arange(32, device="cuda") + offset) % 3 + 1).float().reshape( + 32, 1 + ) / 128 + deferred = {} + with loading_scope(changed): + load_mimo_v2_qkv_proj_weight( + WEIGHT_NAME, projection.weight, full, ckpt_tp, qkv_proj=destination + ) + load_mimo_v2_qkv_proj_weight( + SCALE_NAME, + projection.weight_scale_inv, + scale, + ckpt_tp, + deferred_scale_inv=deferred, + qkv_proj=destination, + ) + assert SCALE_NAME in deferred + assert deferred[SCALE_NAME].data_ptr() != scale.data_ptr() + torch.testing.assert_close(deferred[SCALE_NAME], scale, rtol=0, atol=0) + params = {WEIGHT_NAME: projection.weight, SCALE_NAME: projection.weight_scale_inv} + config = SimpleNamespace( + num_attention_heads=64, num_key_value_heads=32, head_dim=32, v_head_dim=16 + ) + with loading_scope(changed): + _resolve_deferred_qkv_scale_inv( + params, deferred, ckpt_tp, config=config, model=model + ) + first = rank_size(projection)[0] * (ckpt_tp // rank_size(projection)[1]) + last = first + ckpt_tp // rank_size(projection)[1] + dequant = [] + for index in range(first, last): + shard = full.chunk(ckpt_tp, dim=0)[index].float() + shard_scale = scale.chunk(ckpt_tp, dim=0)[index] + dequant.append( + (shard * shard_scale.repeat_interleave(128, dim=0)[:448]).to(torch.bfloat16) + ) + # Independently assemble the checkpoint Q, K and V windows. + merged = torch.cat( + [ + part + for start, stop in ((0, 256), (256, 384), (384, 448)) + for shard in dequant + for part in (shard[start:stop],) + ], + dim=0, + ) + expected_weight = torch.empty_like(projection.weight) + expected_scale = torch.empty_like(projection.weight_scale_inv) + for start in range(0, merged.shape[0], 128): + block = merged[start : start + 128].float() + divisor = block.abs().max().clamp_min(1e-12) / 448 + expected_weight[start : start + 128] = ( + (block / divisor).clamp(-448, 448).to(torch.float8_e4m3fn) + ) + expected_scale[start // 128, 0] = divisor + torch.testing.assert_close( + projection.weight.view(torch.uint8), + expected_weight.view(torch.uint8), + rtol=0, + atol=0, + ) + torch.testing.assert_close( + projection.weight_scale_inv, expected_scale, rtol=0, atol=0 + ) + return projection.weight, projection.weight_scale_inv + + +@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU") +class TestMiMoFusedQkvLoaderLayout(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def check_loads(self, changed, wrapped=False): + for dp in (1, 2): + for rank in range(4): + reset_context() + publish( + ServerArgs( + model_path="dummy", + device="cuda", + tp_size=4, + attn_dp_size=dp, + fp8_gemm_runner_backend="triton", + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + for dtype in (torch.float32, torch.bfloat16): + _, projection = build_projection(dtype) + if wrapped: + projection = BaseLayerWithLoRA(projection, Mock()) + for ckpt_tp, sharded in ( + (None, True), + (None, False), + (rank_size(unwrap_lora_layer(projection))[1], True), + (rank_size(unwrap_lora_layer(projection))[1], False), + (8, False), + ): + for offset in (0, 11): + load_fused( + projection, + changed=changed, + ckpt_tp=ckpt_tp, + sharded=sharded, + offset=offset, + ) + model, projection = build_projection(fp8=True) + if wrapped: + projection = BaseLayerWithLoRA(projection, Mock()) + model.model.layers[0].self_attn.qkv_proj = projection + for offset in (0, 11): + load_deferred(model, projection, changed=changed, offset=offset) + + def test_native_projection_after_scope_exit(self): + self.check_loads(True) + + def test_wrapped_projection_and_deferred_scales_after_scope_exit(self): + self.check_loads(True, wrapped=True) + + def check_main_and_mtp_callers(self, wrapped=False): + for dp in (1, 2): + for rank in range(4): + reset_context() + publish( + ServerArgs( + model_path="dummy", device="cuda", tp_size=4, attn_dp_size=dp + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + for model_type in (MiMoV2ForCausalLM, MiMoV2MTP): + root, projection = build_projection() + model = model_type.__new__(model_type) + torch.nn.Module.__init__(model) + model.config = SimpleNamespace( + n_routed_experts=0, + encoder_only=False, + tie_word_embeddings=False, + ) + model.quant_config = None + model._is_multimodal = False + model.model = root.model + if wrapped: + model.model.layers[0].self_attn.qkv_proj = BaseLayerWithLoRA( + projection, Mock() + ) + name = WEIGHT_NAME + if model_type is MiMoV2MTP: + model.model.mtp_block = model.model.layers[0] + del model.model.layers + name = "model.mtp.layers.0.self_attn.qkv_proj.weight" + full = values((3584, 128), projection.weight.dtype) + expected = full.chunk(rank_size(projection)[1], dim=0)[ + rank_size(projection)[0] + ] + for changed in (False, True): + with loading_scope(changed): + model.load_weights([(name, full)]) + torch.testing.assert_close( + projection.weight, expected, rtol=0, atol=0 + ) + + def test_main_and_mtp_callers_pass_the_native_projection(self): + self.check_main_and_mtp_callers() + + def test_main_and_mtp_callers_unwrap_the_projection(self): + self.check_main_and_mtp_callers(wrapped=True) + + def test_rank_change_without_changing_partition_width(self): + publish( + ServerArgs(model_path="dummy", device="cuda", tp_size=4, attn_dp_size=2), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + _, projection = build_projection() + model, fp8_projection = build_projection(fp8=True) + with parallel_scope(tp_rank=0, attn_tp_rank=0, attn_dp_rank=0, moe_tp_rank=0): + load_fused(projection, ckpt_tp=8) + load_deferred(model, fp8_projection) + + +if __name__ == "__main__": + unittest.main()