Files
sglang/python/sglang/srt/models/mimo_v2.py
T

1801 lines
67 KiB
Python

# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import logging
import math
import re
from array import array
from functools import partial
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
import torch
import torch.nn.functional as F
from torch import nn
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStatePacker,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
)
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
CuteDSLFusion,
MoeDeferredFinalize,
install_cutedsl_fusion,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
LinearParallelGroup,
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
resolve_linear_parallel_group,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
get_moe_runner_backend,
)
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.topk import (
TopK,
TopKOutputChecker,
TopKOutputFormat,
is_expert_remapping_enabled,
)
from sglang.srt.layers.moe.utils import (
RoutingMethodType,
is_deepep_class_backend,
sum_post_experts_output,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
Mxfp4FlashinferTrtllmMoEMethod,
)
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.rotary_embedding.base import RotaryEmbedding
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.managers.mm_utils import (
general_mm_embed_routine,
pad_mm_tokens,
)
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalInputs,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_loader.weight_utils import (
default_weight_loader,
kv_cache_scales_loader,
)
from sglang.srt.models.mimo_audio import AudioEncoderMixin, MiMoAudioEncoderConfig
from sglang.srt.models.mimo_vl import MiMoVisionTransformer, MiMoVLVisionConfig
from sglang.srt.runtime_context import (
get_exec,
get_flags,
get_forward,
get_parallel,
)
from sglang.srt.utils import (
LazyValue,
add_prefix,
ceil_align,
is_non_idle_and_non_empty,
make_pp_layers,
)
MiMoV2Config = None
logger = logging.getLogger(__name__)
def load_mimo_v2_qkv_proj_weight(
name,
param,
loaded_weight,
expected_fused_tp_size: Optional[int] = None,
deferred_scale_inv: Optional[Dict[str, torch.Tensor]] = None,
*,
qkv_proj: QKVParallelLinear,
):
qkv_proj = unwrap_lora_layer(qkv_proj)
tp_rank, tp_size = get_group_rank_size(qkv_proj.tp_group)
ckpt_tp = expected_fused_tp_size if expected_fused_tp_size is not None else tp_size
if ckpt_tp == tp_size and loaded_weight.shape == param.shape:
default_weight_loader(param, loaded_weight)
return
if expected_fused_tp_size is not None and expected_fused_tp_size % tp_size != 0:
raise ValueError(
f"MiMoV2 fused qkv_proj checkpoint is TP={expected_fused_tp_size}-"
f"interleaved; got attention tp_size={tp_size} while loading {name}."
)
is_scale_inv = "weight_scale_inv" in name
if is_scale_inv and ckpt_tp != tp_size:
if deferred_scale_inv is not None:
deferred_scale_inv[name] = loaded_weight.clone()
return
raise ValueError(
f"qkv_proj scale_inv {name}: shape mismatch "
f"{tuple(loaded_weight.shape)} vs {tuple(param.shape)} "
f"due to block quantization ceiling; pass deferred_scale_inv dict"
)
if loaded_weight.ndim != param.ndim or loaded_weight.shape[1:] != param.shape[1:]:
raise ValueError(
f"qkv_proj weight {name}: unexpected shape {tuple(loaded_weight.shape)}; "
f"expected sharded {tuple(param.shape)}"
)
if tp_size == ckpt_tp:
fused_shape = (param.shape[0] * tp_size, *param.shape[1:])
if tuple(loaded_weight.shape) != fused_shape:
raise ValueError(
f"qkv_proj weight {name}: unexpected shape "
f"{tuple(loaded_weight.shape)}; expected fused {fused_shape} "
f"or sharded {tuple(param.shape)}"
)
default_weight_loader(param, loaded_weight.chunk(tp_size, dim=0)[tp_rank])
else:
shards_per_rank = ckpt_tp // tp_size
shards = loaded_weight.chunk(ckpt_tp, dim=0)
merged = torch.cat(
shards[tp_rank * shards_per_rank : (tp_rank + 1) * shards_per_rank],
dim=0,
)
default_weight_loader(param, merged)
def _get_ckpt_qkv_shard_sizes(config, layer_name, ckpt_tp):
m = re.search(r"layers\.(\d+)\.", layer_name)
if m is None:
return None
layer_id = int(m.group(1))
pattern = getattr(config, "hybrid_layer_pattern", None)
is_swa = pattern is not None and pattern[layer_id] == 1
if is_swa:
nh = config.swa_num_attention_heads
nkv = config.swa_num_key_value_heads
hd = config.swa_head_dim
vhd = getattr(config, "swa_v_head_dim", hd)
else:
nh = config.num_attention_heads
nkv = config.num_key_value_heads
hd = config.head_dim
vhd = getattr(config, "v_head_dim", hd)
q_per_shard = (nh // ckpt_tp) * hd
k_per_shard = max(1, nkv // ckpt_tp) * hd
v_per_shard = max(1, nkv // ckpt_tp) * vhd
return (q_per_shard, k_per_shard, v_per_shard)
def _deinterleave_qkv_shards(shards, q_per_shard, k_per_shard, v_per_shard):
all_q, all_k, all_v = [], [], []
for s in shards:
all_q.append(s[:q_per_shard])
all_k.append(s[q_per_shard : q_per_shard + k_per_shard])
all_v.append(s[q_per_shard + k_per_shard :])
return torch.cat(all_q + all_k + all_v, dim=0)
def _resolve_deferred_qkv_scale_inv(
params_dict: Dict[str, torch.nn.Parameter],
deferred_scale_inv: Dict[str, torch.Tensor],
expected_fused_tp_size: int,
block_size: int = 128,
config=None,
*,
model: nn.Module,
):
ckpt_tp = expected_fused_tp_size
for scale_name, ckpt_scale in deferred_scale_inv.items():
weight_name = scale_name.replace(".weight_scale_inv", ".weight")
if weight_name not in params_dict:
raise ValueError(
f"Cannot resolve deferred scale_inv {scale_name}: "
f"weight {weight_name} not found"
)
qkv_proj = unwrap_lora_layer(model.get_submodule(weight_name.rsplit(".", 1)[0]))
tp_rank, tp_size = get_group_rank_size(qkv_proj.tp_group)
shards_per_rank = ckpt_tp // tp_size
weight_param = params_dict[weight_name]
scale_param = params_dict[scale_name]
weight_data = weight_param.data
ckpt_scale_shards = ckpt_scale.chunk(ckpt_tp, dim=0)
my_scale_shards = ckpt_scale_shards[
tp_rank * shards_per_rank : (tp_rank + 1) * shards_per_rank
]
weight_rows = weight_data.shape[0]
rows_per_ckpt_shard = weight_rows // shards_per_rank
block_k = ckpt_scale.shape[1]
device = weight_data.device
dequant_shards = []
for i, shard_scale in enumerate(my_scale_shards):
shard_weight = weight_data[
i * rows_per_ckpt_shard : (i + 1) * rows_per_ckpt_shard
]
shard_scale_f32 = shard_scale.to(dtype=torch.float32, device=device)
scale_expanded = shard_scale_f32.repeat_interleave(
block_size, dim=0
).repeat_interleave(block_size, dim=1)
scale_expanded = scale_expanded[
: shard_weight.shape[0], : shard_weight.shape[1]
]
dequant_shards.append(
(shard_weight.to(torch.float32) * scale_expanded).to(torch.bfloat16)
)
qkv_sizes = (
_get_ckpt_qkv_shard_sizes(config, scale_name, ckpt_tp)
if config is not None
else None
)
if qkv_sizes is not None and shards_per_rank > 1:
merged_bf16 = _deinterleave_qkv_shards(dequant_shards, *qkv_sizes)
else:
merged_bf16 = torch.cat(dequant_shards, dim=0)
n, k = merged_bf16.shape
n_pad = ceil_align(n, block_size)
k_pad = ceil_align(k, block_size)
padded = torch.zeros(
n_pad, k_pad, dtype=merged_bf16.dtype, device=merged_bf16.device
)
padded[:n, :k] = merged_bf16
blocks = padded.view(
n_pad // block_size, block_size, k_pad // block_size, block_size
)
amax = blocks.abs().float().amax(dim=(1, 3), keepdim=True).clamp(min=1e-12)
finfo = torch.finfo(torch.float8_e4m3fn)
new_scale_inv = amax / finfo.max
new_fp8 = (
(blocks.float() / new_scale_inv)
.clamp(min=finfo.min, max=finfo.max)
.to(torch.float8_e4m3fn)
)
n_scale = math.ceil(n / block_size)
k_scale = math.ceil(k / block_size)
weight_param.data.copy_(new_fp8.view(n_pad, k_pad)[:n, :k].contiguous())
default_weight_loader(
scale_param,
new_scale_inv.view(n_pad // block_size, k_pad // block_size)[
:n_scale, :k_scale
].contiguous(),
)
logger.info(
f"Resolved deferred qkv scale_inv {scale_name}: "
f"dequant {shards_per_rank} shards -> requant "
f"({n_scale}, {k_scale})"
+ (f", de-interleaved QKV {qkv_sizes}" if qkv_sizes else "")
)
class MiMoV2MLP(nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
reduce_results: bool = True,
prefix: str = "",
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.is_replicated = parallel_group == "replicated"
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
parallel_group=parallel_group,
)
if hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
)
self.act_fn = SiluAndMul()
def forward(
self,
x,
forward_batch: ForwardBatch = None,
):
if self.is_replicated and x.shape[0] == 0:
return x
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
class MoEGate(nn.Module):
def __init__(
self,
config,
quant_config,
prefix: str = "",
is_nextn: bool = False,
):
super().__init__()
self.is_nextn = is_nextn
self.dtype = getattr(torch, getattr(config, "moe_router_dtype", "float32"))
self.weight = nn.Parameter(
torch.empty((config.n_routed_experts, config.hidden_size), dtype=self.dtype)
)
if config.topk_method == "noaux_tc":
correction_bias_dtype = (
torch.bfloat16
if quant_config is not None
and quant_config.get_name() == "modelopt_fp4"
and get_moe_runner_backend().is_flashinfer_trtllm()
else torch.float32
)
self.e_score_correction_bias = nn.Parameter(
torch.empty((config.n_routed_experts), dtype=correction_bias_dtype)
)
else:
self.e_score_correction_bias = None
def forward(self, hidden_states):
if self.dtype != torch.float32 and hidden_states.is_cuda:
return torch.mm(
hidden_states.to(self.dtype),
self.weight.t(),
out_dtype=torch.float32,
)
logits = F.linear(hidden_states.to(self.dtype), self.weight, None)
return logits.to(torch.float32)
class MiMoV2MoE(nn.Module):
def __init__(
self,
config: MiMoV2Config,
layer_id: int,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
is_nextn: bool = False,
):
super().__init__()
self.tp_size = get_parallel().tp_size
self.config = config
self.layer_id = layer_id
if self.tp_size > config.n_routed_experts:
raise ValueError(
f"Tensor parallel size {self.tp_size} is greater than "
f"the number of experts {config.n_routed_experts}."
)
if config.hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {config.hidden_act}. "
"Only silu is supported for now."
)
self.gate = MoEGate(
config=config,
quant_config=quant_config,
prefix=add_prefix("gate", prefix),
is_nextn=is_nextn,
)
routing_method_type = (
RoutingMethodType.DeepSeekV3
if self.gate.e_score_correction_bias is not None and config.norm_topk_prob
else None
)
experts_type = get_moe_impl_class(quant_config)
self.experts = experts_type(
num_experts=config.n_routed_experts
+ get_exec().moe.ep_num_redundant_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
layer_id=self.layer_id,
quant_config=quant_config,
routed_scaling_factor=1.0,
routing_method_type=routing_method_type,
prefix=add_prefix("experts", prefix),
)
self.topk = TopK(
top_k=config.num_experts_per_tok,
layer_id=self.layer_id,
renormalize=config.norm_topk_prob,
use_grouped_topk=True,
num_expert_group=config.n_group,
topk_group=config.topk_group,
correction_bias=self.gate.e_score_correction_bias,
is_fp4_experts=getattr(quant_config, "is_fp4_experts", False),
scoring_func=config.scoring_func,
quant_config=quant_config,
routed_scaling_factor=1.0,
apply_routed_scaling_factor_on_output=self.experts.should_fuse_routed_scaling_factor_in_topk,
# Some Fp4 MoE backends require the output format to be bypassed but the MTP layers are unquantized
# and requires the output format to be standard. We use quant_config to determine the output format.
output_format=TopKOutputFormat.STANDARD if quant_config is None else None,
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
uses_mxfp4_runner = isinstance(
self.experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod
)
self._uses_expert_remap = (
not get_flags().moe.in_speculative_scope and is_expert_remapping_enabled()
)
use_bypassed_topk = (
uses_cutedsl_ar_fusion()
and not get_flags().moe.in_speculative_scope
and not self._uses_expert_remap
and not get_exec().features.enable_return_routed_experts
and get_exec().moe.expert_distribution_recorder_mode is None
and not envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
and not envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.get()
and uses_mxfp4_runner
and get_moe_a2a_backend().is_none()
and self.experts.quant_method.configure_bypassed_topk(self.topk.topk_config)
)
if uses_mxfp4_runner:
self.topk.topk_config.output_format = (
TopKOutputFormat.BYPASSED
if use_bypassed_topk
else TopKOutputFormat.STANDARD
)
self.supports_deferred_finalize = (
use_bypassed_topk and envs.SGLANG_ENABLE_MOE_DEFERRED_FINALIZE.get()
)
# todo : implement tbo forward needed
self._enable_a2a_moe = (
is_deepep_class_backend() or get_moe_a2a_backend().is_ascend_fuseep()
)
if self._enable_a2a_moe:
# TODO: we will support tp < ep in the future
self.ep_size = get_parallel().moe_ep_size
self.num_experts = (
config.n_routed_experts + get_exec().moe.ep_num_redundant_experts
)
self.renormalize = config.norm_topk_prob
self.topk_group = config.topk_group
self.num_expert_group = config.n_group
self.correction_bias = (
self.gate.e_score_correction_bias.data
if self.gate.e_score_correction_bias is not None
else None
)
def get_moe_weights(self):
return [
x.data
for name, x in self.experts.named_parameters()
if name not in ["correction_bias"]
]
def forward(
self,
hidden_states: torch.Tensor,
forward_batch: Optional[ForwardBatch] = None,
) -> torch.Tensor | MoeDeferredFinalize:
if not self._enable_a2a_moe:
return self.forward_normal(hidden_states)
else:
return self.forward_deepep(hidden_states, forward_batch)
def forward_normal(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | MoeDeferredFinalize:
if hidden_states.shape[0] > 0:
router_logits = self.gate(hidden_states)
topk_output = self.topk(
hidden_states,
router_logits,
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
# Precomputed routing keeps FP32 weights and finalizes before the reduction.
if get_forward().defer_moe_finalize and TopKOutputChecker.format_is_bypassed(
topk_output
):
deferred = self.experts.forward_deferred_finalize(
hidden_states, topk_output
)
return MoeDeferredFinalize.from_flashinfer(
deferred,
gated_shared_output=None,
m=hidden_states.shape[0],
reduce=sum_post_experts_output,
)
final_hidden_states = self.experts(hidden_states, topk_output)
return final_hidden_states
def forward_deepep(
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
if hidden_states.shape[0] > 0:
# router_logits: (num_tokens, n_experts)
router_logits = self.gate(hidden_states)
topk_output = self.topk(
hidden_states,
router_logits,
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
final_hidden_states = self.experts(
hidden_states=hidden_states, topk_output=topk_output
)
return final_hidden_states
def op_gate(self, state):
if is_non_idle_and_non_empty(
state.forward_batch.forward_mode, state.hidden_states_mlp_input
):
# router_logits: (num_tokens, n_experts)
state.router_logits = self.gate(state.hidden_states_mlp_input)
else:
state.router_logits = None
def op_select_experts(self, state):
router_logits = state.pop("router_logits")
hidden_states = state.hidden_states_mlp_input
if router_logits is not None:
with get_global_expert_distribution_recorder().with_current_layer(
self.layer_id
):
state.topk_output = self.topk(
hidden_states=hidden_states,
router_logits=router_logits,
)
else:
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
def op_dispatch_a(self, state):
if self.ep_size > 1:
self.experts.dispatcher.dispatch_a(
hidden_states=state.pop("hidden_states_mlp_input"),
topk_output=state.pop("topk_output"),
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_dispatch_b(self, state):
if self.ep_size > 1:
with get_global_expert_distribution_recorder().with_current_layer(
self.layer_id
):
state.dispatch_output = self.experts.dispatcher.dispatch_b(
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_experts(self, state):
state.combine_input = self.experts.run_moe_core(
dispatch_output=state.dispatch_output,
)
def op_combine_a(self, state):
if self.ep_size > 1:
self.experts.dispatcher.combine_a(
combine_input=state.pop("combine_input"),
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
state.pop("dispatch_output")
def op_combine_b(self, state):
if self.ep_size > 1:
state.hidden_states_after_combine = self.experts.dispatcher.combine_b(
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_output(self, state):
state.hidden_states_mlp_output = state.pop("hidden_states_after_combine")
class MiMoV2Attention(nn.Module):
def __init__(
self,
hidden_size: int,
num_heads: int,
num_kv_heads: int,
head_dim: Optional[int] = None,
v_head_dim: Optional[int] = None,
v_scale: Optional[float] = None,
sliding_window_size: int = -1, # if is -1 ,normal attention,else ,window attention
attention_bias: bool = False,
attention_sink_bias: bool = False,
layer_id: int = 0,
rope_theta: float = 1000000,
rope_scaling: Optional[Dict[str, Any]] = None,
max_position_embeddings: int = 32768,
quant_config: Optional[QuantizationConfig] = None,
partial_rotary_factor: float = 1.0,
prefix: str = "",
) -> None:
super().__init__()
self.hidden_size = hidden_size
attn_tp_size = get_parallel().attn_tp_size
self.total_num_heads = num_heads
assert self.total_num_heads % attn_tp_size == 0
self.num_heads = self.total_num_heads // attn_tp_size
self.total_num_kv_heads = num_kv_heads
if self.total_num_kv_heads >= attn_tp_size:
# Number of KV heads is greater than TP size, so we partition
# the KV heads across multiple tensor parallel GPUs.
assert self.total_num_kv_heads % attn_tp_size == 0
else:
# Number of KV heads is less than TP size, so we replicate
# the KV heads across multiple tensor parallel GPUs.
assert attn_tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // attn_tp_size)
self.head_dim = head_dim
self.v_head_dim = v_head_dim if v_head_dim is not None else head_dim
self.q_size = self.num_heads * self.head_dim
self.k_size = self.num_kv_heads * self.head_dim
self.v_size = self.num_kv_heads * self.v_head_dim
self.v_scale = v_scale
self.scaling = self.head_dim**-0.5
self.qkv_proj = QKVParallelLinear(
hidden_size,
self.head_dim,
self.total_num_heads,
self.total_num_kv_heads,
v_head_size=self.v_head_dim,
bias=attention_bias,
quant_config=quant_config,
parallel_group="attn_tp",
prefix=add_prefix("qkv_proj", prefix),
skip_block_quant_check=True,
)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.v_head_dim,
hidden_size,
bias=False,
quant_config=quant_config,
parallel_group="attn_tp",
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)
self.rotary_emb = get_rope(
self.head_dim,
rotary_dim=self.head_dim,
max_position=max_position_embeddings,
base=rope_theta,
rope_scaling=rope_scaling,
partial_rotary_factor=partial_rotary_factor,
)
self.attn = RadixAttention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
layer_id=layer_id,
v_head_dim=self.v_head_dim,
sliding_window_size=sliding_window_size, # if is -1 ,normal attention,else ,window attention
quant_config=quant_config,
prefix=add_prefix("attn", prefix),
)
self.attention_sink_bias = (
torch.nn.Parameter(torch.empty(self.num_heads), requires_grad=False)
if attention_sink_bias
else None
)
def op_prepare(self, state):
state.attn_intermediate_state = self.forward_prepare(
positions=state.positions,
hidden_states=state.pop("hidden_states_after_comm_pre_attn"),
forward_batch=state.forward_batch,
)
def op_core(self, state):
state.hidden_states_after_attn = self.forward_core(
state.pop("attn_intermediate_state")
)
def forward_prepare(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
):
if hidden_states.shape[0] == 0:
return hidden_states, forward_batch, None
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
if self.v_scale is not None:
v = v * self.v_scale
inner_state = q, k, v, forward_batch
return None, forward_batch, inner_state
def forward_core(self, intermediate_state):
hidden_states, forward_batch, inner_state = intermediate_state
if inner_state is None:
return hidden_states
attn_output = self.attn(
*inner_state,
sinks=self.attention_sink_bias,
)
output, _ = self.o_proj(attn_output)
return output
def _forward_rope_store(self, qkv, positions, forward_batch):
from sglang.srt.model_executor.forward_context import get_attn_backend
query = self.rotary_emb.try_bf16_rope_store(
qkv,
positions,
get_attn_backend().get_fused_kv_write_buffers(self.attn, forward_batch),
value_scale=1.0 if self.v_scale is None else self.v_scale,
)
if query is None:
return None
return self.attn(
query,
None,
None,
forward_batch,
save_kv_cache=False,
sinks=self.attention_sink_bias,
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
if (
forward_batch.forward_mode.is_target_verify()
and qkv.shape[0] > 0
and isinstance(self.rotary_emb, RotaryEmbedding)
and not torch.compiler.is_compiling()
):
fused = self._forward_rope_store(qkv, positions, forward_batch)
if fused is not None:
return self.o_proj(fused)[0]
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
# [t, h, dr]
q, k = self.rotary_emb(positions, q, k)
# [t, h, d]
if self.v_scale is not None:
v = v * self.v_scale
attn_output = self.attn(q, k, v, forward_batch, sinks=self.attention_sink_bias)
output, _ = self.o_proj(attn_output)
return output
class MiMoV2DecoderLayer(nn.Module):
def __init__(
self,
config: MiMoV2Config,
layer_id: int = 0,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
super().__init__()
self.config = config
self.hidden_size = config.hidden_size
self.layer_id = layer_id
rope_theta = getattr(config, "rope_theta", 10000)
rope_scaling = getattr(config, "rope_scaling", None)
# In v5, rope_scaling is a property alias for rope_parameters and returns
# a standardized dict even when there's no actual scaling. Treat the
# "default" (no-op) type as None so factory.py uses plain RotaryEmbedding.
if (
isinstance(rope_scaling, dict)
and rope_scaling.get("rope_type") == "default"
):
rope_scaling = None
max_position_embeddings = getattr(
config,
"context_len",
getattr(config, "max_position_embeddings", 32768),
)
if self.is_swa_layer():
self.self_attn = MiMoV2Attention(
hidden_size=self.hidden_size,
num_heads=config.swa_num_attention_heads,
num_kv_heads=config.swa_num_key_value_heads,
head_dim=config.swa_head_dim,
v_head_dim=getattr(config, "swa_v_head_dim", None),
v_scale=getattr(config, "attention_value_scale", None),
sliding_window_size=config.sliding_window_size,
attention_bias=config.attention_bias,
attention_sink_bias=getattr(
config, "add_swa_attention_sink_bias", False
),
layer_id=layer_id,
rope_theta=getattr(config, "swa_rope_theta", rope_theta),
rope_scaling=rope_scaling,
max_position_embeddings=max_position_embeddings,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=add_prefix("self_attn", prefix),
)
else:
self.self_attn = MiMoV2Attention(
hidden_size=self.hidden_size,
num_heads=self.config.num_attention_heads,
num_kv_heads=config.num_key_value_heads,
head_dim=config.head_dim,
v_head_dim=getattr(config, "v_head_dim", None),
v_scale=getattr(config, "attention_value_scale", None),
sliding_window_size=-1, # normal attention
attention_bias=config.attention_bias,
attention_sink_bias=getattr(
config, "add_full_attention_sink_bias", False
),
layer_id=layer_id,
rope_theta=rope_theta,
rope_scaling=rope_scaling,
max_position_embeddings=max_position_embeddings,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=add_prefix("self_attn", prefix),
)
self.is_layer_sparse = self.is_moe_layer(config, layer_id)
if self.is_layer_sparse:
self.mlp = MiMoV2MoE(
config=config,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
layer_id=layer_id,
)
else:
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = MiMoV2MLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
parallel_group=mlp_parallel_group,
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.layernorm_epsilon
)
attn, ffn = self.stage_facts(config, layer_id)
fusions = (
CuteDSLFusion()
if uses_cutedsl_ar_fusion() and not get_flags().moe.in_speculative_scope
else None
)
self.attn_boundary, self.ffn_boundary = append_stages(
(attn, self.input_layernorm, {"fusions": fusions}),
(ffn, self.post_attention_layernorm, {"fusions": fusions}),
)
@classmethod
def stage_facts(cls, config, layer_id: int):
"""The stages the layer at ``layer_id`` declares, in order: the model's
shared declaration function, which the layer declares with too (see
make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls.is_moe_layer(config, layer_id),
next_layer_sparse=cls.is_moe_layer(config, layer_id + 1),
),
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
) -> torch.Tensor:
# Self Attention
hidden_states = self.attn_boundary.prepare(
hidden_states,
forward_batch,
capture_gathered=captured_last_layer_outputs,
)
if hidden_states.shape[0] != 0:
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
@staticmethod
def is_moe_layer(config, layer_idx: int) -> bool:
return (
hasattr(config, "moe_layer_freq")
and 0 <= layer_idx < len(config.moe_layer_freq)
and not isinstance(config.moe_layer_freq, int)
and config.moe_layer_freq[layer_idx]
)
def is_swa_layer(self) -> bool:
return self.config.hybrid_layer_pattern[self.layer_id] == 1
def op_comm_prepare_attn(
self,
state,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
hidden_states, forward_batch
)
state.update(
dict(
forward_batch=forward_batch,
positions=positions,
tbo_subbatch_index=tbo_subbatch_index,
)
)
def op_comm_prepare_mlp(self, state):
hidden_states = self.attn_boundary.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_boundary.complete_now(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
forward_batch=state.forward_batch,
tbo_subbatch_index=state.tbo_subbatch_index,
)
state.clear(
expect_keys={
"positions",
"forward_batch",
"tbo_subbatch_index",
}
)
return output
class MiMoV2Model(nn.Module):
def __init__(
self,
config: MiMoV2Config,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
decoder_layer_type: type[nn.Module] = MiMoV2DecoderLayer,
) -> None:
super().__init__()
self.config = config
self.padding_idx = getattr(config, "pad_token_id", None)
self.vocab_size = config.vocab_size
self.pp_group = get_parallel().pp_group
self._kv_cache_parallel_layout = resolve_linear_parallel_group("attn_tp")
self.layers_to_capture = []
if self.pp_group.is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
use_attn_tp_group=is_dp_attention_enabled(),
prefix=add_prefix("embed_tokens", prefix),
)
else:
self.embed_tokens = PPMissingLayer()
# Use the provided decoder layer type or default to MiMoV2DecoderLayer
decoder_layer_type = decoder_layer_type or MiMoV2DecoderLayer
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
layer_fn=lambda idx, prefix: decoder_layer_type(
layer_id=idx,
config=config,
quant_config=quant_config,
prefix=prefix,
),
prefix=add_prefix("layers", prefix),
stage_facts=partial(decoder_layer_type.stage_facts, config),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
else:
self.norm = PPMissingLayer(return_tuple=True)
if uses_cutedsl_ar_fusion():
install_cutedsl_fusion(
self.layers[self.start_layer : self.end_layer],
hidden_size=config.hidden_size,
top_k=config.num_experts_per_tok,
rms_epsilon=config.layernorm_epsilon,
can_defer_finalize=lambda layer: (
isinstance(layer.mlp, MiMoV2MoE)
and layer.mlp.supports_deferred_finalize
),
label="MiMo",
include_shared_expert=False,
)
def get_input_embedding(self, input_ids: torch.Tensor) -> torch.Tensor:
if hasattr(self.config, "scale_emb"):
return self.get_input_embeddings()(input_ids) * self.config.scale_emb
else:
return self.get_input_embeddings()(input_ids)
def get_input_embeddings(self) -> nn.Embedding:
return self.embed_tokens
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: torch.Tensor = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> Union[torch.Tensor, PPProxyTensors]:
if self.pp_group.is_first_rank:
if input_embeds is None:
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = AuxHiddenStatePacker(len(self.layers_to_capture))
if forward_batch.can_run_tbo and not self.layers_to_capture:
tbo_start_layer = self.start_layer
tbo_end_layer = self.end_layer
# skip first layer for TBO when starting from layer 0
if self.start_layer == 0:
layer = self.layers[0]
hidden_states = layer(positions, hidden_states, forward_batch)
tbo_start_layer = tbo_start_layer + 1
hidden_states = model_forward_stages(
layers=self.layers[tbo_start_layer:tbo_end_layer],
enable_tbo=True,
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
)
else:
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
hidden_states = layer(
positions,
hidden_states,
forward_batch,
captured_last_layer_outputs=aux_hidden_states
if i in self.layers_to_capture
else None,
)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
# A draft targeting the final layer ("after layer
# num_hidden_layers-1") maps to capture index num_hidden_layers,
# past the layer loop; capture the pre-norm output here instead.
if (
self.pp_group.is_last_rank
and self.config.num_hidden_layers in self.layers_to_capture
):
aux_hidden_states.append(
residual_batch.snapshot(hidden_states, forward_batch)
)
hidden_states_before_norm = None
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
if hidden_states.shape[0] > 0:
if forward_batch.return_hidden_states_before_norm:
hidden_states_before_norm = residual_batch.snapshot(
hidden_states, forward_batch
)
hidden_states = residual_batch.final_norm(
hidden_states, forward_batch, self.norm
)
if len(aux_hidden_states) == 0:
return hidden_states, hidden_states_before_norm
return hidden_states, hidden_states_before_norm, aux_hidden_states.finalize()
# If this function is called, it should always initialize KV cache scale
# factors (or else raise an exception). Thus, handled exceptions should
# make sure to leave KV cache scale factors in a known good (dummy) state
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
attn_tp_rank, attn_tp_size = self._kv_cache_parallel_layout
for layer_idx, scaling_factor in kv_cache_scales_loader(
quantization_param_path,
attn_tp_rank,
attn_tp_size,
self.config.num_hidden_layers,
self.config.__class__.model_type,
):
if not isinstance(self.layers[layer_idx], nn.Identity):
layer_self_attn = self.layers[layer_idx].self_attn
if hasattr(layer_self_attn.attn, "k_scale"):
layer_self_attn.attn.k_scale = scaling_factor
layer_self_attn.attn.v_scale = scaling_factor
else:
raise RuntimeError(
"Self attention has no KV cache scaling factor attribute!"
)
class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
# BitandBytes specific attributes
default_bitsandbytes_target_modules = [
".gate_proj.",
".down_proj.",
".up_proj.",
".q_proj.",
".k_proj.",
".v_proj.",
".o_proj.",
]
bitsandbytes_stacked_params_mapping = {
# shard_name, weight_name, index
"q_proj": ("qkv_proj", 0),
"k_proj": ("qkv_proj", 1),
"v_proj": ("qkv_proj", 2),
"gate_proj": ("gate_up_proj", 0),
"up_proj": ("gate_up_proj", 1),
}
# Prefixes for weight routing in encoder_only/language_only modes
_LANGUAGE_WEIGHT_PREFIXES = ("model.", "lm_head.")
_VISION_WEIGHT_PREFIXES = ("visual.", "vision_model.")
# ``audio_`` already covers ``audio_encoder.`` so a single prefix is enough.
_AUDIO_WEIGHT_PREFIXES = ("audio_",)
_AUDIO_WEIGHT_SUBSTRING = "speech_embeddings"
def __init__(
self,
config: MiMoV2Config,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
super().__init__()
self.pp_group = get_parallel().pp_group
self.config = config
self.quant_config = quant_config
self._encoder_processor = None # lazy-created in preprocess_mm_for_encoder
if not self.config.encoder_only:
self.model = MiMoV2Model(
config, quant_config=quant_config, prefix=add_prefix("model", prefix)
)
if self.pp_group.is_last_rank:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=add_prefix("lm_head", prefix),
use_attn_tp_group=get_parallel().enable_dp_lm_head,
)
else:
self.lm_head = PPMissingLayer()
else:
self.model = None
self.lm_head = None
self.logits_processor = (
LogitsProcessor(config) if not self.config.encoder_only else None
)
self.capture_aux_hidden_states = False
vision_config = getattr(config, "vision_config", None)
audio_config = getattr(config, "audio_config", None)
self._is_multimodal = vision_config is not None and audio_config is not None
# Always build vision/audio encoders so P can fall back to local
# encoding when the EPD encoder is unreachable.
if self._is_multimodal:
if hasattr(vision_config, "to_dict"):
vision_config = vision_config.to_dict()
if hasattr(audio_config, "to_dict"):
audio_config = audio_config.to_dict()
self.visual = MiMoVisionTransformer(
MiMoVLVisionConfig.from_dict(vision_config),
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
quant_config=None,
prefix=add_prefix("visual", prefix),
)
self.build_audio_encoder(MiMoAudioEncoderConfig(**audio_config))
self._routed_experts_weights_of_layer = LazyValue(
lambda: (
{
layer_id: layer.mlp.get_moe_weights()
for layer_id, layer in enumerate(self.model.layers)
if isinstance(layer.mlp, MiMoV2MoE)
}
if self.model is not None
else {}
)
)
@property
def routed_experts_weights_of_layer(self):
return self._routed_experts_weights_of_layer.value
def get_input_embedding(self, input_ids: torch.Tensor) -> torch.Tensor:
assert self.model is not None, (
"get_input_embedding() is not available in encoder_only mode"
)
return self.model.get_input_embedding(input_ids)
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return pad_mm_tokens(input_ids, mm_inputs)
def preprocess_mm_for_encoder(self, mm_data, modality, config):
if self._encoder_processor is None:
from sglang.srt.multimodal.processors.mimo_v2 import MiMoProcessor
self._encoder_processor = MiMoProcessor.from_hf_config(
self.config, mm_config=config
)
return self._encoder_processor.preprocess_for_encoder(mm_data, modality)
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
pixel_values = torch.cat([item.feature for item in items], dim=0).type(
self.visual.dtype
)
image_grid_thw = torch.cat([item.image_grid_thw for item in items], dim=0)
assert pixel_values.dim() == 2, pixel_values.dim()
assert image_grid_thw.dim() == 2, image_grid_thw.dim()
return self.visual(pixel_values, grid_thw=image_grid_thw)
def get_video_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
pixel_values = torch.cat([item.feature for item in items], dim=0).type(
self.visual.dtype
)
video_grid_thw = torch.cat([item.video_grid_thw for item in items], dim=0)
assert pixel_values.dim() == 2, pixel_values.dim()
assert video_grid_thw.dim() == 2, video_grid_thw.dim()
return self.visual(pixel_values, grid_thw=video_grid_thw)
@torch.inference_mode()
def encode_video_audio(self, mm_inputs: Dict) -> Optional[torch.Tensor]:
# EPD-side hook: encode audio tracks pulled from videos and trim to the
# interleaved per-video segments produced by MiMoProcessor (segment
# starts / lens / per_video_num_units). Returns None if there is no
# audio to encode. The server passes the result through to the receiver
# under aux_data["video_audio_embedding"].
import numpy as np
audio_features = mm_inputs.get("video_audio_features")
if not audio_features:
return None
def _as_tensor(data):
if isinstance(data, torch.Tensor):
return data
if isinstance(data, np.ndarray):
return torch.tensor(data)
if isinstance(data, list) and data and isinstance(data[0], np.ndarray):
return torch.tensor(np.array(data))
if isinstance(data, list) and data and isinstance(data[0], (int, float)):
return torch.tensor(data)
return data
audio_feature_lens = mm_inputs["video_audio_feature_lens"]
audio_item = MultimodalDataItem.from_dict(
{
"modality": Modality.AUDIO,
"feature": _as_tensor(audio_features),
}
)
audio_item.set("audio_feature_lens", _as_tensor(audio_feature_lens))
audio_embedding = self.get_audio_feature([audio_item]).cpu()
if audio_embedding.ndim != 2:
audio_embedding = audio_embedding.reshape(-1, audio_embedding.shape[-1])
segment_lens_flat = mm_inputs["video_audio_segment_lens_flat"]
segment_starts_flat = mm_inputs["video_audio_segment_starts_flat"]
per_video_num_units = mm_inputs["video_audio_per_video_num_units"]
per_video_audio_token_lens = (
audio_feature_lens.tolist()
if hasattr(audio_feature_lens, "tolist")
else list(audio_feature_lens)
)
trimmed_chunks = []
emb_offset = 0
unit_idx = 0
audio_video_idx = 0
for num_units in per_video_num_units:
if num_units <= 0:
continue
vid_audio_len = per_video_audio_token_lens[audio_video_idx]
for _ in range(num_units):
start = segment_starts_flat[unit_idx]
seg_len = segment_lens_flat[unit_idx]
trimmed_chunks.append(
audio_embedding[emb_offset + start : emb_offset + start + seg_len]
)
unit_idx += 1
emb_offset += vid_audio_len
audio_video_idx += 1
return (
torch.cat(trimmed_chunks, dim=0) if trimmed_chunks else audio_embedding[:0]
)
def get_input_embeddings(self) -> Optional[nn.Embedding]:
return self.model.embed_tokens if self.model is not None else None
@torch.no_grad()
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: torch.Tensor = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> torch.Tensor:
assert not self.config.encoder_only, (
"forward() should not be called in encoder_only mode"
)
aux_hidden_states = None
# Multimodal embedding must run even when aux hidden states are captured
# (DFLASH/MTP). Taking the capture branch first skipped image/audio
# feature placement and left placeholder tokens as plain text embeddings.
if self._is_multimodal:
model_out = general_mm_embed_routine(
input_ids=input_ids,
forward_batch=forward_batch,
language_model=self.model,
multimodal_model=self,
positions=positions,
pp_proxy_tensors=pp_proxy_tensors,
)
else:
model_out = self.model(
input_ids,
positions,
forward_batch,
input_embeds,
pp_proxy_tensors=pp_proxy_tensors,
)
if self.capture_aux_hidden_states:
hidden_states, hidden_states_before_norm, aux_hidden_states = model_out
else:
hidden_states, hidden_states_before_norm = model_out
if self.pp_group.is_last_rank:
return self.logits_processor(
input_ids,
hidden_states,
self.lm_head,
forward_batch,
hidden_states_before_norm=hidden_states_before_norm,
aux_hidden_states=aux_hidden_states,
)
else:
return hidden_states
@property
def start_layer(self):
return self.model.start_layer if self.model is not None else 0
@property
def end_layer(self):
return self.model.end_layer if self.model is not None else 0
def set_dflash_layers_to_capture(self, layer_ids: List[int]):
if not self.pp_group.is_last_rank:
return
if layer_ids is None:
raise ValueError(
"DFLASH requires explicit layer_ids for aux hidden capture."
)
self.capture_aux_hidden_states = True
# target_layer_ids are "after layer X" ids; capture before layer X+1,
# matching the draft's extract_context_feature (offset=1).
self.model.layers_to_capture = [val + 1 for val in layer_ids]
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
stacked_params_mapping_vit = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = DeepEPMoE.make_expert_params_mapping(
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts,
)
params_dict = dict(self.named_parameters())
skipped_mtp_weights = False
deferred_qkv_scale_inv: Dict[str, torch.Tensor] = {}
for name, loaded_weight in weights:
is_vision_weight = name.startswith(self._VISION_WEIGHT_PREFIXES)
is_audio_weight = (
name.startswith(self._AUDIO_WEIGHT_PREFIXES)
or self._AUDIO_WEIGHT_SUBSTRING in name
)
if not self._is_multimodal and (is_vision_weight or is_audio_weight):
continue
if self.config.encoder_only and name.startswith(
self._LANGUAGE_WEIGHT_PREFIXES
):
continue
if self._is_multimodal and is_audio_weight:
if name.startswith("audio_encoder."):
name = name[len("audio_encoder.") :]
name = self.remap_audio_weight_name(name)
if "input_local_transformer" not in name:
name = name.replace("self_attn.out_proj", "self_attn.attn.proj")
audio_stacked = False
for param_name, weight_name, shard_id in [
("self_attn.attn.qkv_proj", "self_attn.q_proj", "q"),
("self_attn.attn.qkv_proj", "self_attn.k_proj", "k"),
("self_attn.attn.qkv_proj", "self_attn.v_proj", "v"),
]:
if weight_name in name:
name = name.replace(weight_name, param_name)
if name not in params_dict:
break
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
audio_stacked = True
break
if audio_stacked:
continue
if name not in params_dict:
logger.warning(
f"Audio param {name} not found in params_dict, skipping"
)
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if self._AUDIO_WEIGHT_SUBSTRING in name:
weight_loader(param, loaded_weight[: param.shape[0], :])
else:
weight_loader(param, loaded_weight)
continue
if self._is_multimodal and "visual" in name:
name = name.replace("vision_model.", "")
name = name.replace(r"attn.qkv.", r"attn.qkv_proj.")
match_stacked_vit = False
for param_name, weight_name, shard_id in stacked_params_mapping_vit:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
match_stacked_vit = True
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
match_stacked_vit = True
break
if match_stacked_vit:
continue
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
if name.endswith("patch_embed.proj.weight"):
patch_embed = self.get_submodule(name.rsplit(".", 2)[0])
if hasattr(patch_embed, "sync_proj_weight_linear_format"):
patch_embed.sync_proj_weight_linear_format()
continue
layer_id = get_layer_id(name)
if (
layer_id is not None
and hasattr(self.model, "start_layer")
and (
layer_id < self.model.start_layer
or layer_id >= self.model.end_layer
)
):
continue
if "rotary_emb.inv_freq" in name or "projector" in name:
continue
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
# Models trained using ColossalAI may include these tensors in
# the checkpoint. Skip them.
continue
if self.config.tie_word_embeddings and "lm_head.weight" in name:
if self.pp_group.world_size > 1 and self.pp_group.is_last_rank:
# Handle pp weight tying here
# find the embed_tokens.weight in the weights
embed_token_weights = next(
filter(lambda x: x[0] == "model.embed_tokens.weight", weights)
)[1]
loaded_weight = embed_token_weights
else:
continue
if "mtp" in name:
if not skipped_mtp_weights:
logger.info(
"Skipping draft-only MiMo-V2 MTP weights while loading the "
"target model; MiMoV2MTP loads these weights in the draft "
"model runner."
)
skipped_mtp_weights = True
continue
if ".mlp.experts." in name and loaded_weight.dtype == torch.uint8:
if name.endswith(".weight_scale"):
name = name + "_inv"
loaded_weight = torch.exp2(loaded_weight.to(torch.float32) - 127.0)
elif name.endswith(".weight"):
loaded_weight = loaded_weight.view(torch.int8)
# Support fused qkv_proj checkpoint (Pro format)
if "qkv_proj" in name:
if name in params_dict:
param = params_dict[name]
expected_fused_tp_size = get_mimo_v2_fused_qkv_expected_tp_size(
self.config
)
load_mimo_v2_qkv_proj_weight(
name,
param,
loaded_weight,
expected_fused_tp_size,
deferred_scale_inv=deferred_qkv_scale_inv,
qkv_proj=self.get_submodule(name.rsplit(".", 1)[0]),
)
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if (
"compression_attention" in name
or "hybrid_softmax_attention" in name
or "compressed_softmax_attn" in name
):
continue
if weight_name not in name:
continue
if ("mlp.experts." in name) and name not in params_dict:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# mxfp4 ckpts store expert scales without the `_inv` suffix,
# while Fp8MoEMethod registers them as *_weight_scale_inv.
if name.endswith("weight_scale") and (name + "_inv" in params_dict):
name = name + "_inv"
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if name in params_dict.keys():
param = params_dict[name]
if "attention_sink_bias" in name:
projection = unwrap_lora_layer(
self.get_submodule(name.rsplit(".", 1)[0]).qkv_proj
)
start = (
get_group_rank_size(projection.tp_group)[0]
* param.numel()
)
param.data.copy_(
loaded_weight[start : start + param.numel()]
)
else:
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
else:
logger.warning(f"Parameter {name} not found in params_dict")
if deferred_qkv_scale_inv:
expected_fused_tp_size = get_mimo_v2_fused_qkv_expected_tp_size(self.config)
_resolve_deferred_qkv_scale_inv(
params_dict,
deferred_qkv_scale_inv,
expected_fused_tp_size,
config=self.config,
model=self,
)
def get_embed_and_head(self):
assert self.model is not None and self.lm_head is not None, (
"get_embed_and_head() is not available in encoder_only mode"
)
return self.model.embed_tokens.weight, self.lm_head.weight
def set_embed_and_head(self, embed, head):
assert self.model is not None and self.lm_head is not None, (
"set_embed_and_head() is not available in encoder_only mode"
)
del self.model.embed_tokens.weight
del self.lm_head.weight
self.model.embed_tokens.weight = embed
self.lm_head.weight = head
torch.cuda.empty_cache()
torch.cuda.synchronize()
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
if self.model is not None:
self.model.load_kv_cache_scales(quantization_param_path)
@classmethod
def get_model_config_for_expert_location(cls, config):
return ModelConfigForExpertLocation(
num_layers=config.num_hidden_layers,
num_logical_experts=getattr(config, "n_routed_experts", 1),
num_groups=getattr(config, "n_group", None),
)
# Keep the old Flash architecture name loadable while new configs use MiMoV2ForCausalLM.
class MiMoV2FlashForCausalLM(MiMoV2ForCausalLM):
pass
EntryClass = [MiMoV2ForCausalLM, MiMoV2FlashForCausalLM]