Co-authored-by: Aditya Suresh <152752311+adi-suresh01@users.noreply.github.com>
1801 lines
67 KiB
Python
1801 lines
67 KiB
Python
# Copyright 2023-2024 SGLang Team
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# ==============================================================================
|
|
|
|
import logging
|
|
import math
|
|
import re
|
|
from array import array
|
|
from functools import partial
|
|
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
|
|
|
import torch
|
|
import torch.nn.functional as F
|
|
from torch import nn
|
|
|
|
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
|
|
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
|
|
from sglang.srt.distributed.utils import get_group_rank_size
|
|
from sglang.srt.environ import envs
|
|
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
|
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
|
from sglang.srt.layers.activation import SiluAndMul
|
|
from sglang.srt.layers.aux_hidden_states import (
|
|
AuxHiddenStateAccumulator,
|
|
AuxHiddenStatePacker,
|
|
)
|
|
from sglang.srt.layers.dp_attention import (
|
|
is_dp_attention_enabled,
|
|
)
|
|
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
|
from sglang.srt.layers.layer_boundary import (
|
|
append_stages,
|
|
declare_attn,
|
|
declare_ffn,
|
|
is_dense_ffn_fully_dp,
|
|
)
|
|
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
|
|
CuteDSLFusion,
|
|
MoeDeferredFinalize,
|
|
install_cutedsl_fusion,
|
|
)
|
|
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
|
from sglang.srt.layers.layernorm import RMSNorm
|
|
from sglang.srt.layers.linear import (
|
|
LinearParallelGroup,
|
|
MergedColumnParallelLinear,
|
|
QKVParallelLinear,
|
|
RowParallelLinear,
|
|
resolve_linear_parallel_group,
|
|
)
|
|
from sglang.srt.layers.logits_processor import LogitsProcessor
|
|
from sglang.srt.layers.moe import (
|
|
get_moe_a2a_backend,
|
|
get_moe_runner_backend,
|
|
)
|
|
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
|
|
from sglang.srt.layers.moe.topk import (
|
|
TopK,
|
|
TopKOutputChecker,
|
|
TopKOutputFormat,
|
|
is_expert_remapping_enabled,
|
|
)
|
|
from sglang.srt.layers.moe.utils import (
|
|
RoutingMethodType,
|
|
is_deepep_class_backend,
|
|
sum_post_experts_output,
|
|
)
|
|
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
|
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
|
|
Mxfp4FlashinferTrtllmMoEMethod,
|
|
)
|
|
from sglang.srt.layers.radix_attention import RadixAttention
|
|
from sglang.srt.layers.rotary_embedding import get_rope
|
|
from sglang.srt.layers.rotary_embedding.base import RotaryEmbedding
|
|
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
|
from sglang.srt.layers.vocab_parallel_embedding import (
|
|
ParallelLMHead,
|
|
VocabParallelEmbedding,
|
|
)
|
|
from sglang.srt.lora.layers import unwrap_lora_layer
|
|
from sglang.srt.managers.mm_utils import (
|
|
general_mm_embed_routine,
|
|
pad_mm_tokens,
|
|
)
|
|
from sglang.srt.managers.schedule_batch import (
|
|
Modality,
|
|
MultimodalDataItem,
|
|
MultimodalInputs,
|
|
)
|
|
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
|
from sglang.srt.model_loader.weight_utils import (
|
|
default_weight_loader,
|
|
kv_cache_scales_loader,
|
|
)
|
|
from sglang.srt.models.mimo_audio import AudioEncoderMixin, MiMoAudioEncoderConfig
|
|
from sglang.srt.models.mimo_vl import MiMoVisionTransformer, MiMoVLVisionConfig
|
|
from sglang.srt.runtime_context import (
|
|
get_exec,
|
|
get_flags,
|
|
get_forward,
|
|
get_parallel,
|
|
)
|
|
from sglang.srt.utils import (
|
|
LazyValue,
|
|
add_prefix,
|
|
ceil_align,
|
|
is_non_idle_and_non_empty,
|
|
make_pp_layers,
|
|
)
|
|
|
|
MiMoV2Config = None
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def load_mimo_v2_qkv_proj_weight(
|
|
name,
|
|
param,
|
|
loaded_weight,
|
|
expected_fused_tp_size: Optional[int] = None,
|
|
deferred_scale_inv: Optional[Dict[str, torch.Tensor]] = None,
|
|
*,
|
|
qkv_proj: QKVParallelLinear,
|
|
):
|
|
qkv_proj = unwrap_lora_layer(qkv_proj)
|
|
tp_rank, tp_size = get_group_rank_size(qkv_proj.tp_group)
|
|
ckpt_tp = expected_fused_tp_size if expected_fused_tp_size is not None else tp_size
|
|
|
|
if ckpt_tp == tp_size and loaded_weight.shape == param.shape:
|
|
default_weight_loader(param, loaded_weight)
|
|
return
|
|
|
|
if expected_fused_tp_size is not None and expected_fused_tp_size % tp_size != 0:
|
|
raise ValueError(
|
|
f"MiMoV2 fused qkv_proj checkpoint is TP={expected_fused_tp_size}-"
|
|
f"interleaved; got attention tp_size={tp_size} while loading {name}."
|
|
)
|
|
|
|
is_scale_inv = "weight_scale_inv" in name
|
|
|
|
if is_scale_inv and ckpt_tp != tp_size:
|
|
if deferred_scale_inv is not None:
|
|
deferred_scale_inv[name] = loaded_weight.clone()
|
|
return
|
|
raise ValueError(
|
|
f"qkv_proj scale_inv {name}: shape mismatch "
|
|
f"{tuple(loaded_weight.shape)} vs {tuple(param.shape)} "
|
|
f"due to block quantization ceiling; pass deferred_scale_inv dict"
|
|
)
|
|
|
|
if loaded_weight.ndim != param.ndim or loaded_weight.shape[1:] != param.shape[1:]:
|
|
raise ValueError(
|
|
f"qkv_proj weight {name}: unexpected shape {tuple(loaded_weight.shape)}; "
|
|
f"expected sharded {tuple(param.shape)}"
|
|
)
|
|
|
|
if tp_size == ckpt_tp:
|
|
fused_shape = (param.shape[0] * tp_size, *param.shape[1:])
|
|
if tuple(loaded_weight.shape) != fused_shape:
|
|
raise ValueError(
|
|
f"qkv_proj weight {name}: unexpected shape "
|
|
f"{tuple(loaded_weight.shape)}; expected fused {fused_shape} "
|
|
f"or sharded {tuple(param.shape)}"
|
|
)
|
|
default_weight_loader(param, loaded_weight.chunk(tp_size, dim=0)[tp_rank])
|
|
else:
|
|
shards_per_rank = ckpt_tp // tp_size
|
|
shards = loaded_weight.chunk(ckpt_tp, dim=0)
|
|
merged = torch.cat(
|
|
shards[tp_rank * shards_per_rank : (tp_rank + 1) * shards_per_rank],
|
|
dim=0,
|
|
)
|
|
default_weight_loader(param, merged)
|
|
|
|
|
|
def _get_ckpt_qkv_shard_sizes(config, layer_name, ckpt_tp):
|
|
m = re.search(r"layers\.(\d+)\.", layer_name)
|
|
if m is None:
|
|
return None
|
|
layer_id = int(m.group(1))
|
|
|
|
pattern = getattr(config, "hybrid_layer_pattern", None)
|
|
is_swa = pattern is not None and pattern[layer_id] == 1
|
|
|
|
if is_swa:
|
|
nh = config.swa_num_attention_heads
|
|
nkv = config.swa_num_key_value_heads
|
|
hd = config.swa_head_dim
|
|
vhd = getattr(config, "swa_v_head_dim", hd)
|
|
else:
|
|
nh = config.num_attention_heads
|
|
nkv = config.num_key_value_heads
|
|
hd = config.head_dim
|
|
vhd = getattr(config, "v_head_dim", hd)
|
|
|
|
q_per_shard = (nh // ckpt_tp) * hd
|
|
k_per_shard = max(1, nkv // ckpt_tp) * hd
|
|
v_per_shard = max(1, nkv // ckpt_tp) * vhd
|
|
return (q_per_shard, k_per_shard, v_per_shard)
|
|
|
|
|
|
def _deinterleave_qkv_shards(shards, q_per_shard, k_per_shard, v_per_shard):
|
|
all_q, all_k, all_v = [], [], []
|
|
for s in shards:
|
|
all_q.append(s[:q_per_shard])
|
|
all_k.append(s[q_per_shard : q_per_shard + k_per_shard])
|
|
all_v.append(s[q_per_shard + k_per_shard :])
|
|
return torch.cat(all_q + all_k + all_v, dim=0)
|
|
|
|
|
|
def _resolve_deferred_qkv_scale_inv(
|
|
params_dict: Dict[str, torch.nn.Parameter],
|
|
deferred_scale_inv: Dict[str, torch.Tensor],
|
|
expected_fused_tp_size: int,
|
|
block_size: int = 128,
|
|
config=None,
|
|
*,
|
|
model: nn.Module,
|
|
):
|
|
ckpt_tp = expected_fused_tp_size
|
|
|
|
for scale_name, ckpt_scale in deferred_scale_inv.items():
|
|
weight_name = scale_name.replace(".weight_scale_inv", ".weight")
|
|
if weight_name not in params_dict:
|
|
raise ValueError(
|
|
f"Cannot resolve deferred scale_inv {scale_name}: "
|
|
f"weight {weight_name} not found"
|
|
)
|
|
|
|
qkv_proj = unwrap_lora_layer(model.get_submodule(weight_name.rsplit(".", 1)[0]))
|
|
tp_rank, tp_size = get_group_rank_size(qkv_proj.tp_group)
|
|
shards_per_rank = ckpt_tp // tp_size
|
|
weight_param = params_dict[weight_name]
|
|
scale_param = params_dict[scale_name]
|
|
weight_data = weight_param.data
|
|
|
|
ckpt_scale_shards = ckpt_scale.chunk(ckpt_tp, dim=0)
|
|
my_scale_shards = ckpt_scale_shards[
|
|
tp_rank * shards_per_rank : (tp_rank + 1) * shards_per_rank
|
|
]
|
|
|
|
weight_rows = weight_data.shape[0]
|
|
rows_per_ckpt_shard = weight_rows // shards_per_rank
|
|
block_k = ckpt_scale.shape[1]
|
|
|
|
device = weight_data.device
|
|
dequant_shards = []
|
|
for i, shard_scale in enumerate(my_scale_shards):
|
|
shard_weight = weight_data[
|
|
i * rows_per_ckpt_shard : (i + 1) * rows_per_ckpt_shard
|
|
]
|
|
shard_scale_f32 = shard_scale.to(dtype=torch.float32, device=device)
|
|
scale_expanded = shard_scale_f32.repeat_interleave(
|
|
block_size, dim=0
|
|
).repeat_interleave(block_size, dim=1)
|
|
scale_expanded = scale_expanded[
|
|
: shard_weight.shape[0], : shard_weight.shape[1]
|
|
]
|
|
dequant_shards.append(
|
|
(shard_weight.to(torch.float32) * scale_expanded).to(torch.bfloat16)
|
|
)
|
|
|
|
qkv_sizes = (
|
|
_get_ckpt_qkv_shard_sizes(config, scale_name, ckpt_tp)
|
|
if config is not None
|
|
else None
|
|
)
|
|
if qkv_sizes is not None and shards_per_rank > 1:
|
|
merged_bf16 = _deinterleave_qkv_shards(dequant_shards, *qkv_sizes)
|
|
else:
|
|
merged_bf16 = torch.cat(dequant_shards, dim=0)
|
|
|
|
n, k = merged_bf16.shape
|
|
n_pad = ceil_align(n, block_size)
|
|
k_pad = ceil_align(k, block_size)
|
|
padded = torch.zeros(
|
|
n_pad, k_pad, dtype=merged_bf16.dtype, device=merged_bf16.device
|
|
)
|
|
padded[:n, :k] = merged_bf16
|
|
blocks = padded.view(
|
|
n_pad // block_size, block_size, k_pad // block_size, block_size
|
|
)
|
|
amax = blocks.abs().float().amax(dim=(1, 3), keepdim=True).clamp(min=1e-12)
|
|
finfo = torch.finfo(torch.float8_e4m3fn)
|
|
new_scale_inv = amax / finfo.max
|
|
new_fp8 = (
|
|
(blocks.float() / new_scale_inv)
|
|
.clamp(min=finfo.min, max=finfo.max)
|
|
.to(torch.float8_e4m3fn)
|
|
)
|
|
|
|
n_scale = math.ceil(n / block_size)
|
|
k_scale = math.ceil(k / block_size)
|
|
weight_param.data.copy_(new_fp8.view(n_pad, k_pad)[:n, :k].contiguous())
|
|
default_weight_loader(
|
|
scale_param,
|
|
new_scale_inv.view(n_pad // block_size, k_pad // block_size)[
|
|
:n_scale, :k_scale
|
|
].contiguous(),
|
|
)
|
|
|
|
logger.info(
|
|
f"Resolved deferred qkv scale_inv {scale_name}: "
|
|
f"dequant {shards_per_rank} shards -> requant "
|
|
f"({n_scale}, {k_scale})"
|
|
+ (f", de-interleaved QKV {qkv_sizes}" if qkv_sizes else "")
|
|
)
|
|
|
|
|
|
class MiMoV2MLP(nn.Module):
|
|
def __init__(
|
|
self,
|
|
hidden_size: int,
|
|
intermediate_size: int,
|
|
hidden_act: str,
|
|
quant_config: Optional[QuantizationConfig] = None,
|
|
reduce_results: bool = True,
|
|
prefix: str = "",
|
|
*,
|
|
parallel_group: LinearParallelGroup = "tp",
|
|
) -> None:
|
|
super().__init__()
|
|
self.is_replicated = parallel_group == "replicated"
|
|
|
|
self.gate_up_proj = MergedColumnParallelLinear(
|
|
hidden_size,
|
|
[intermediate_size] * 2,
|
|
bias=False,
|
|
quant_config=quant_config,
|
|
prefix=add_prefix("gate_up_proj", prefix),
|
|
parallel_group=parallel_group,
|
|
)
|
|
self.down_proj = RowParallelLinear(
|
|
intermediate_size,
|
|
hidden_size,
|
|
bias=False,
|
|
quant_config=quant_config,
|
|
reduce_results=reduce_results,
|
|
prefix=add_prefix("down_proj", prefix),
|
|
parallel_group=parallel_group,
|
|
)
|
|
if hidden_act != "silu":
|
|
raise ValueError(
|
|
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
|
|
)
|
|
self.act_fn = SiluAndMul()
|
|
|
|
def forward(
|
|
self,
|
|
x,
|
|
forward_batch: ForwardBatch = None,
|
|
):
|
|
if self.is_replicated and x.shape[0] == 0:
|
|
return x
|
|
|
|
gate_up, _ = self.gate_up_proj(x)
|
|
x = self.act_fn(gate_up)
|
|
x, _ = self.down_proj(x)
|
|
return x
|
|
|
|
|
|
class MoEGate(nn.Module):
|
|
def __init__(
|
|
self,
|
|
config,
|
|
quant_config,
|
|
prefix: str = "",
|
|
is_nextn: bool = False,
|
|
):
|
|
super().__init__()
|
|
self.is_nextn = is_nextn
|
|
self.dtype = getattr(torch, getattr(config, "moe_router_dtype", "float32"))
|
|
self.weight = nn.Parameter(
|
|
torch.empty((config.n_routed_experts, config.hidden_size), dtype=self.dtype)
|
|
)
|
|
if config.topk_method == "noaux_tc":
|
|
correction_bias_dtype = (
|
|
torch.bfloat16
|
|
if quant_config is not None
|
|
and quant_config.get_name() == "modelopt_fp4"
|
|
and get_moe_runner_backend().is_flashinfer_trtllm()
|
|
else torch.float32
|
|
)
|
|
self.e_score_correction_bias = nn.Parameter(
|
|
torch.empty((config.n_routed_experts), dtype=correction_bias_dtype)
|
|
)
|
|
else:
|
|
self.e_score_correction_bias = None
|
|
|
|
def forward(self, hidden_states):
|
|
if self.dtype != torch.float32 and hidden_states.is_cuda:
|
|
return torch.mm(
|
|
hidden_states.to(self.dtype),
|
|
self.weight.t(),
|
|
out_dtype=torch.float32,
|
|
)
|
|
|
|
logits = F.linear(hidden_states.to(self.dtype), self.weight, None)
|
|
return logits.to(torch.float32)
|
|
|
|
|
|
class MiMoV2MoE(nn.Module):
|
|
def __init__(
|
|
self,
|
|
config: MiMoV2Config,
|
|
layer_id: int,
|
|
quant_config: Optional[QuantizationConfig] = None,
|
|
prefix: str = "",
|
|
is_nextn: bool = False,
|
|
):
|
|
super().__init__()
|
|
self.tp_size = get_parallel().tp_size
|
|
|
|
self.config = config
|
|
self.layer_id = layer_id
|
|
|
|
if self.tp_size > config.n_routed_experts:
|
|
raise ValueError(
|
|
f"Tensor parallel size {self.tp_size} is greater than "
|
|
f"the number of experts {config.n_routed_experts}."
|
|
)
|
|
|
|
if config.hidden_act != "silu":
|
|
raise ValueError(
|
|
f"Unsupported activation: {config.hidden_act}. "
|
|
"Only silu is supported for now."
|
|
)
|
|
|
|
self.gate = MoEGate(
|
|
config=config,
|
|
quant_config=quant_config,
|
|
prefix=add_prefix("gate", prefix),
|
|
is_nextn=is_nextn,
|
|
)
|
|
|
|
routing_method_type = (
|
|
RoutingMethodType.DeepSeekV3
|
|
if self.gate.e_score_correction_bias is not None and config.norm_topk_prob
|
|
else None
|
|
)
|
|
experts_type = get_moe_impl_class(quant_config)
|
|
self.experts = experts_type(
|
|
num_experts=config.n_routed_experts
|
|
+ get_exec().moe.ep_num_redundant_experts,
|
|
top_k=config.num_experts_per_tok,
|
|
hidden_size=config.hidden_size,
|
|
intermediate_size=config.moe_intermediate_size,
|
|
layer_id=self.layer_id,
|
|
quant_config=quant_config,
|
|
routed_scaling_factor=1.0,
|
|
routing_method_type=routing_method_type,
|
|
prefix=add_prefix("experts", prefix),
|
|
)
|
|
|
|
self.topk = TopK(
|
|
top_k=config.num_experts_per_tok,
|
|
layer_id=self.layer_id,
|
|
renormalize=config.norm_topk_prob,
|
|
use_grouped_topk=True,
|
|
num_expert_group=config.n_group,
|
|
topk_group=config.topk_group,
|
|
correction_bias=self.gate.e_score_correction_bias,
|
|
is_fp4_experts=getattr(quant_config, "is_fp4_experts", False),
|
|
scoring_func=config.scoring_func,
|
|
quant_config=quant_config,
|
|
routed_scaling_factor=1.0,
|
|
apply_routed_scaling_factor_on_output=self.experts.should_fuse_routed_scaling_factor_in_topk,
|
|
# Some Fp4 MoE backends require the output format to be bypassed but the MTP layers are unquantized
|
|
# and requires the output format to be standard. We use quant_config to determine the output format.
|
|
output_format=TopKOutputFormat.STANDARD if quant_config is None else None,
|
|
)
|
|
self.topk.bind_runner_backend(
|
|
self.experts.runner_backend,
|
|
routing_method_type=self.experts.routing_method_type,
|
|
)
|
|
|
|
uses_mxfp4_runner = isinstance(
|
|
self.experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod
|
|
)
|
|
self._uses_expert_remap = (
|
|
not get_flags().moe.in_speculative_scope and is_expert_remapping_enabled()
|
|
)
|
|
use_bypassed_topk = (
|
|
uses_cutedsl_ar_fusion()
|
|
and not get_flags().moe.in_speculative_scope
|
|
and not self._uses_expert_remap
|
|
and not get_exec().features.enable_return_routed_experts
|
|
and get_exec().moe.expert_distribution_recorder_mode is None
|
|
and not envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
|
|
and not envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.get()
|
|
and uses_mxfp4_runner
|
|
and get_moe_a2a_backend().is_none()
|
|
and self.experts.quant_method.configure_bypassed_topk(self.topk.topk_config)
|
|
)
|
|
if uses_mxfp4_runner:
|
|
self.topk.topk_config.output_format = (
|
|
TopKOutputFormat.BYPASSED
|
|
if use_bypassed_topk
|
|
else TopKOutputFormat.STANDARD
|
|
)
|
|
self.supports_deferred_finalize = (
|
|
use_bypassed_topk and envs.SGLANG_ENABLE_MOE_DEFERRED_FINALIZE.get()
|
|
)
|
|
|
|
# todo : implement tbo forward needed
|
|
self._enable_a2a_moe = (
|
|
is_deepep_class_backend() or get_moe_a2a_backend().is_ascend_fuseep()
|
|
)
|
|
if self._enable_a2a_moe:
|
|
# TODO: we will support tp < ep in the future
|
|
self.ep_size = get_parallel().moe_ep_size
|
|
self.num_experts = (
|
|
config.n_routed_experts + get_exec().moe.ep_num_redundant_experts
|
|
)
|
|
self.renormalize = config.norm_topk_prob
|
|
self.topk_group = config.topk_group
|
|
self.num_expert_group = config.n_group
|
|
self.correction_bias = (
|
|
self.gate.e_score_correction_bias.data
|
|
if self.gate.e_score_correction_bias is not None
|
|
else None
|
|
)
|
|
|
|
def get_moe_weights(self):
|
|
return [
|
|
x.data
|
|
for name, x in self.experts.named_parameters()
|
|
if name not in ["correction_bias"]
|
|
]
|
|
|
|
def forward(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
forward_batch: Optional[ForwardBatch] = None,
|
|
) -> torch.Tensor | MoeDeferredFinalize:
|
|
if not self._enable_a2a_moe:
|
|
return self.forward_normal(hidden_states)
|
|
else:
|
|
return self.forward_deepep(hidden_states, forward_batch)
|
|
|
|
def forward_normal(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
) -> torch.Tensor | MoeDeferredFinalize:
|
|
|
|
if hidden_states.shape[0] > 0:
|
|
router_logits = self.gate(hidden_states)
|
|
topk_output = self.topk(
|
|
hidden_states,
|
|
router_logits,
|
|
)
|
|
else:
|
|
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
|
|
|
# Precomputed routing keeps FP32 weights and finalizes before the reduction.
|
|
if get_forward().defer_moe_finalize and TopKOutputChecker.format_is_bypassed(
|
|
topk_output
|
|
):
|
|
deferred = self.experts.forward_deferred_finalize(
|
|
hidden_states, topk_output
|
|
)
|
|
return MoeDeferredFinalize.from_flashinfer(
|
|
deferred,
|
|
gated_shared_output=None,
|
|
m=hidden_states.shape[0],
|
|
reduce=sum_post_experts_output,
|
|
)
|
|
|
|
final_hidden_states = self.experts(hidden_states, topk_output)
|
|
|
|
return final_hidden_states
|
|
|
|
def forward_deepep(
|
|
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
|
) -> torch.Tensor:
|
|
if hidden_states.shape[0] > 0:
|
|
# router_logits: (num_tokens, n_experts)
|
|
router_logits = self.gate(hidden_states)
|
|
topk_output = self.topk(
|
|
hidden_states,
|
|
router_logits,
|
|
)
|
|
else:
|
|
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
|
|
|
final_hidden_states = self.experts(
|
|
hidden_states=hidden_states, topk_output=topk_output
|
|
)
|
|
|
|
return final_hidden_states
|
|
|
|
def op_gate(self, state):
|
|
if is_non_idle_and_non_empty(
|
|
state.forward_batch.forward_mode, state.hidden_states_mlp_input
|
|
):
|
|
# router_logits: (num_tokens, n_experts)
|
|
state.router_logits = self.gate(state.hidden_states_mlp_input)
|
|
else:
|
|
state.router_logits = None
|
|
|
|
def op_select_experts(self, state):
|
|
router_logits = state.pop("router_logits")
|
|
hidden_states = state.hidden_states_mlp_input
|
|
if router_logits is not None:
|
|
with get_global_expert_distribution_recorder().with_current_layer(
|
|
self.layer_id
|
|
):
|
|
state.topk_output = self.topk(
|
|
hidden_states=hidden_states,
|
|
router_logits=router_logits,
|
|
)
|
|
else:
|
|
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
|
|
|
|
def op_dispatch_a(self, state):
|
|
if self.ep_size > 1:
|
|
self.experts.dispatcher.dispatch_a(
|
|
hidden_states=state.pop("hidden_states_mlp_input"),
|
|
topk_output=state.pop("topk_output"),
|
|
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
|
)
|
|
|
|
def op_dispatch_b(self, state):
|
|
if self.ep_size > 1:
|
|
with get_global_expert_distribution_recorder().with_current_layer(
|
|
self.layer_id
|
|
):
|
|
state.dispatch_output = self.experts.dispatcher.dispatch_b(
|
|
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
|
)
|
|
|
|
def op_experts(self, state):
|
|
state.combine_input = self.experts.run_moe_core(
|
|
dispatch_output=state.dispatch_output,
|
|
)
|
|
|
|
def op_combine_a(self, state):
|
|
if self.ep_size > 1:
|
|
self.experts.dispatcher.combine_a(
|
|
combine_input=state.pop("combine_input"),
|
|
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
|
)
|
|
state.pop("dispatch_output")
|
|
|
|
def op_combine_b(self, state):
|
|
if self.ep_size > 1:
|
|
state.hidden_states_after_combine = self.experts.dispatcher.combine_b(
|
|
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
|
)
|
|
|
|
def op_output(self, state):
|
|
state.hidden_states_mlp_output = state.pop("hidden_states_after_combine")
|
|
|
|
|
|
class MiMoV2Attention(nn.Module):
|
|
def __init__(
|
|
self,
|
|
hidden_size: int,
|
|
num_heads: int,
|
|
num_kv_heads: int,
|
|
head_dim: Optional[int] = None,
|
|
v_head_dim: Optional[int] = None,
|
|
v_scale: Optional[float] = None,
|
|
sliding_window_size: int = -1, # if is -1 ,normal attention,else ,window attention
|
|
attention_bias: bool = False,
|
|
attention_sink_bias: bool = False,
|
|
layer_id: int = 0,
|
|
rope_theta: float = 1000000,
|
|
rope_scaling: Optional[Dict[str, Any]] = None,
|
|
max_position_embeddings: int = 32768,
|
|
quant_config: Optional[QuantizationConfig] = None,
|
|
partial_rotary_factor: float = 1.0,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
self.hidden_size = hidden_size
|
|
|
|
attn_tp_size = get_parallel().attn_tp_size
|
|
|
|
self.total_num_heads = num_heads
|
|
assert self.total_num_heads % attn_tp_size == 0
|
|
self.num_heads = self.total_num_heads // attn_tp_size
|
|
self.total_num_kv_heads = num_kv_heads
|
|
if self.total_num_kv_heads >= attn_tp_size:
|
|
# Number of KV heads is greater than TP size, so we partition
|
|
# the KV heads across multiple tensor parallel GPUs.
|
|
assert self.total_num_kv_heads % attn_tp_size == 0
|
|
else:
|
|
# Number of KV heads is less than TP size, so we replicate
|
|
# the KV heads across multiple tensor parallel GPUs.
|
|
assert attn_tp_size % self.total_num_kv_heads == 0
|
|
self.num_kv_heads = max(1, self.total_num_kv_heads // attn_tp_size)
|
|
self.head_dim = head_dim
|
|
self.v_head_dim = v_head_dim if v_head_dim is not None else head_dim
|
|
|
|
self.q_size = self.num_heads * self.head_dim
|
|
self.k_size = self.num_kv_heads * self.head_dim
|
|
self.v_size = self.num_kv_heads * self.v_head_dim
|
|
|
|
self.v_scale = v_scale
|
|
|
|
self.scaling = self.head_dim**-0.5
|
|
|
|
self.qkv_proj = QKVParallelLinear(
|
|
hidden_size,
|
|
self.head_dim,
|
|
self.total_num_heads,
|
|
self.total_num_kv_heads,
|
|
v_head_size=self.v_head_dim,
|
|
bias=attention_bias,
|
|
quant_config=quant_config,
|
|
parallel_group="attn_tp",
|
|
prefix=add_prefix("qkv_proj", prefix),
|
|
skip_block_quant_check=True,
|
|
)
|
|
|
|
self.o_proj = RowParallelLinear(
|
|
self.total_num_heads * self.v_head_dim,
|
|
hidden_size,
|
|
bias=False,
|
|
quant_config=quant_config,
|
|
parallel_group="attn_tp",
|
|
reduce_results=False,
|
|
prefix=add_prefix("o_proj", prefix),
|
|
)
|
|
|
|
self.rotary_emb = get_rope(
|
|
self.head_dim,
|
|
rotary_dim=self.head_dim,
|
|
max_position=max_position_embeddings,
|
|
base=rope_theta,
|
|
rope_scaling=rope_scaling,
|
|
partial_rotary_factor=partial_rotary_factor,
|
|
)
|
|
|
|
self.attn = RadixAttention(
|
|
self.num_heads,
|
|
self.head_dim,
|
|
self.scaling,
|
|
num_kv_heads=self.num_kv_heads,
|
|
layer_id=layer_id,
|
|
v_head_dim=self.v_head_dim,
|
|
sliding_window_size=sliding_window_size, # if is -1 ,normal attention,else ,window attention
|
|
quant_config=quant_config,
|
|
prefix=add_prefix("attn", prefix),
|
|
)
|
|
|
|
self.attention_sink_bias = (
|
|
torch.nn.Parameter(torch.empty(self.num_heads), requires_grad=False)
|
|
if attention_sink_bias
|
|
else None
|
|
)
|
|
|
|
def op_prepare(self, state):
|
|
state.attn_intermediate_state = self.forward_prepare(
|
|
positions=state.positions,
|
|
hidden_states=state.pop("hidden_states_after_comm_pre_attn"),
|
|
forward_batch=state.forward_batch,
|
|
)
|
|
|
|
def op_core(self, state):
|
|
state.hidden_states_after_attn = self.forward_core(
|
|
state.pop("attn_intermediate_state")
|
|
)
|
|
|
|
def forward_prepare(
|
|
self,
|
|
positions: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
):
|
|
if hidden_states.shape[0] == 0:
|
|
return hidden_states, forward_batch, None
|
|
qkv, _ = self.qkv_proj(hidden_states)
|
|
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
|
|
|
q, k = self.rotary_emb(positions, q, k)
|
|
if self.v_scale is not None:
|
|
v = v * self.v_scale
|
|
|
|
inner_state = q, k, v, forward_batch
|
|
return None, forward_batch, inner_state
|
|
|
|
def forward_core(self, intermediate_state):
|
|
hidden_states, forward_batch, inner_state = intermediate_state
|
|
if inner_state is None:
|
|
return hidden_states
|
|
attn_output = self.attn(
|
|
*inner_state,
|
|
sinks=self.attention_sink_bias,
|
|
)
|
|
output, _ = self.o_proj(attn_output)
|
|
return output
|
|
|
|
def _forward_rope_store(self, qkv, positions, forward_batch):
|
|
from sglang.srt.model_executor.forward_context import get_attn_backend
|
|
|
|
query = self.rotary_emb.try_bf16_rope_store(
|
|
qkv,
|
|
positions,
|
|
get_attn_backend().get_fused_kv_write_buffers(self.attn, forward_batch),
|
|
value_scale=1.0 if self.v_scale is None else self.v_scale,
|
|
)
|
|
if query is None:
|
|
return None
|
|
return self.attn(
|
|
query,
|
|
None,
|
|
None,
|
|
forward_batch,
|
|
save_kv_cache=False,
|
|
sinks=self.attention_sink_bias,
|
|
)
|
|
|
|
def forward(
|
|
self,
|
|
positions: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
) -> torch.Tensor:
|
|
qkv, _ = self.qkv_proj(hidden_states)
|
|
if (
|
|
forward_batch.forward_mode.is_target_verify()
|
|
and qkv.shape[0] > 0
|
|
and isinstance(self.rotary_emb, RotaryEmbedding)
|
|
and not torch.compiler.is_compiling()
|
|
):
|
|
fused = self._forward_rope_store(qkv, positions, forward_batch)
|
|
if fused is not None:
|
|
return self.o_proj(fused)[0]
|
|
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
|
|
|
# [t, h, dr]
|
|
q, k = self.rotary_emb(positions, q, k)
|
|
# [t, h, d]
|
|
|
|
if self.v_scale is not None:
|
|
v = v * self.v_scale
|
|
attn_output = self.attn(q, k, v, forward_batch, sinks=self.attention_sink_bias)
|
|
output, _ = self.o_proj(attn_output)
|
|
return output
|
|
|
|
|
|
class MiMoV2DecoderLayer(nn.Module):
|
|
def __init__(
|
|
self,
|
|
config: MiMoV2Config,
|
|
layer_id: int = 0,
|
|
quant_config: Optional[QuantizationConfig] = None,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
self.config = config
|
|
self.hidden_size = config.hidden_size
|
|
self.layer_id = layer_id
|
|
|
|
rope_theta = getattr(config, "rope_theta", 10000)
|
|
rope_scaling = getattr(config, "rope_scaling", None)
|
|
# In v5, rope_scaling is a property alias for rope_parameters and returns
|
|
# a standardized dict even when there's no actual scaling. Treat the
|
|
# "default" (no-op) type as None so factory.py uses plain RotaryEmbedding.
|
|
if (
|
|
isinstance(rope_scaling, dict)
|
|
and rope_scaling.get("rope_type") == "default"
|
|
):
|
|
rope_scaling = None
|
|
max_position_embeddings = getattr(
|
|
config,
|
|
"context_len",
|
|
getattr(config, "max_position_embeddings", 32768),
|
|
)
|
|
|
|
if self.is_swa_layer():
|
|
self.self_attn = MiMoV2Attention(
|
|
hidden_size=self.hidden_size,
|
|
num_heads=config.swa_num_attention_heads,
|
|
num_kv_heads=config.swa_num_key_value_heads,
|
|
head_dim=config.swa_head_dim,
|
|
v_head_dim=getattr(config, "swa_v_head_dim", None),
|
|
v_scale=getattr(config, "attention_value_scale", None),
|
|
sliding_window_size=config.sliding_window_size,
|
|
attention_bias=config.attention_bias,
|
|
attention_sink_bias=getattr(
|
|
config, "add_swa_attention_sink_bias", False
|
|
),
|
|
layer_id=layer_id,
|
|
rope_theta=getattr(config, "swa_rope_theta", rope_theta),
|
|
rope_scaling=rope_scaling,
|
|
max_position_embeddings=max_position_embeddings,
|
|
quant_config=quant_config,
|
|
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
|
|
prefix=add_prefix("self_attn", prefix),
|
|
)
|
|
else:
|
|
self.self_attn = MiMoV2Attention(
|
|
hidden_size=self.hidden_size,
|
|
num_heads=self.config.num_attention_heads,
|
|
num_kv_heads=config.num_key_value_heads,
|
|
head_dim=config.head_dim,
|
|
v_head_dim=getattr(config, "v_head_dim", None),
|
|
v_scale=getattr(config, "attention_value_scale", None),
|
|
sliding_window_size=-1, # normal attention
|
|
attention_bias=config.attention_bias,
|
|
attention_sink_bias=getattr(
|
|
config, "add_full_attention_sink_bias", False
|
|
),
|
|
layer_id=layer_id,
|
|
rope_theta=rope_theta,
|
|
rope_scaling=rope_scaling,
|
|
max_position_embeddings=max_position_embeddings,
|
|
quant_config=quant_config,
|
|
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
|
|
prefix=add_prefix("self_attn", prefix),
|
|
)
|
|
|
|
self.is_layer_sparse = self.is_moe_layer(config, layer_id)
|
|
|
|
if self.is_layer_sparse:
|
|
self.mlp = MiMoV2MoE(
|
|
config=config,
|
|
quant_config=quant_config,
|
|
prefix=add_prefix("mlp", prefix),
|
|
layer_id=layer_id,
|
|
)
|
|
else:
|
|
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
|
self.mlp = MiMoV2MLP(
|
|
hidden_size=self.hidden_size,
|
|
intermediate_size=config.intermediate_size,
|
|
hidden_act=config.hidden_act,
|
|
quant_config=quant_config,
|
|
prefix=add_prefix("mlp", prefix),
|
|
parallel_group=mlp_parallel_group,
|
|
reduce_results=False,
|
|
)
|
|
|
|
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
|
|
self.post_attention_layernorm = RMSNorm(
|
|
config.hidden_size, eps=config.layernorm_epsilon
|
|
)
|
|
|
|
attn, ffn = self.stage_facts(config, layer_id)
|
|
fusions = (
|
|
CuteDSLFusion()
|
|
if uses_cutedsl_ar_fusion() and not get_flags().moe.in_speculative_scope
|
|
else None
|
|
)
|
|
self.attn_boundary, self.ffn_boundary = append_stages(
|
|
(attn, self.input_layernorm, {"fusions": fusions}),
|
|
(ffn, self.post_attention_layernorm, {"fusions": fusions}),
|
|
)
|
|
|
|
@classmethod
|
|
def stage_facts(cls, config, layer_id: int):
|
|
"""The stages the layer at ``layer_id`` declares, in order: the model's
|
|
shared declaration function, which the layer declares with too (see
|
|
make_layers)."""
|
|
return (
|
|
declare_attn(),
|
|
declare_ffn(
|
|
sparse=cls.is_moe_layer(config, layer_id),
|
|
next_layer_sparse=cls.is_moe_layer(config, layer_id + 1),
|
|
),
|
|
)
|
|
|
|
def forward(
|
|
self,
|
|
positions: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None,
|
|
) -> torch.Tensor:
|
|
# Self Attention
|
|
hidden_states = self.attn_boundary.prepare(
|
|
hidden_states,
|
|
forward_batch,
|
|
capture_gathered=captured_last_layer_outputs,
|
|
)
|
|
|
|
if hidden_states.shape[0] != 0:
|
|
hidden_states = self.self_attn(
|
|
positions=positions,
|
|
hidden_states=hidden_states,
|
|
forward_batch=forward_batch,
|
|
)
|
|
|
|
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
|
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
|
|
|
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
|
hidden_states = self.mlp(hidden_states, forward_batch)
|
|
hidden_states = ffn_exit.finish(hidden_states)
|
|
|
|
return hidden_states
|
|
|
|
@staticmethod
|
|
def is_moe_layer(config, layer_idx: int) -> bool:
|
|
return (
|
|
hasattr(config, "moe_layer_freq")
|
|
and 0 <= layer_idx < len(config.moe_layer_freq)
|
|
and not isinstance(config.moe_layer_freq, int)
|
|
and config.moe_layer_freq[layer_idx]
|
|
)
|
|
|
|
def is_swa_layer(self) -> bool:
|
|
return self.config.hybrid_layer_pattern[self.layer_id] == 1
|
|
|
|
def op_comm_prepare_attn(
|
|
self,
|
|
state,
|
|
positions: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
tbo_subbatch_index: Optional[int] = None,
|
|
):
|
|
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
|
|
hidden_states, forward_batch
|
|
)
|
|
state.update(
|
|
dict(
|
|
forward_batch=forward_batch,
|
|
positions=positions,
|
|
tbo_subbatch_index=tbo_subbatch_index,
|
|
)
|
|
)
|
|
|
|
def op_comm_prepare_mlp(self, state):
|
|
hidden_states = self.attn_boundary.finish(
|
|
state.pop("hidden_states_after_attn"), state.forward_batch
|
|
)
|
|
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
|
|
hidden_states, state.forward_batch
|
|
)
|
|
|
|
def op_comm_postprocess_layer(self, state):
|
|
hidden_states = self.ffn_boundary.complete_now(
|
|
state.pop("hidden_states_mlp_output"), state.forward_batch
|
|
)
|
|
|
|
output = dict(
|
|
positions=state.positions,
|
|
hidden_states=hidden_states,
|
|
forward_batch=state.forward_batch,
|
|
tbo_subbatch_index=state.tbo_subbatch_index,
|
|
)
|
|
|
|
state.clear(
|
|
expect_keys={
|
|
"positions",
|
|
"forward_batch",
|
|
"tbo_subbatch_index",
|
|
}
|
|
)
|
|
return output
|
|
|
|
|
|
class MiMoV2Model(nn.Module):
|
|
def __init__(
|
|
self,
|
|
config: MiMoV2Config,
|
|
quant_config: Optional[QuantizationConfig] = None,
|
|
prefix: str = "",
|
|
decoder_layer_type: type[nn.Module] = MiMoV2DecoderLayer,
|
|
) -> None:
|
|
super().__init__()
|
|
self.config = config
|
|
self.padding_idx = getattr(config, "pad_token_id", None)
|
|
self.vocab_size = config.vocab_size
|
|
self.pp_group = get_parallel().pp_group
|
|
self._kv_cache_parallel_layout = resolve_linear_parallel_group("attn_tp")
|
|
self.layers_to_capture = []
|
|
|
|
if self.pp_group.is_first_rank:
|
|
self.embed_tokens = VocabParallelEmbedding(
|
|
config.vocab_size,
|
|
config.hidden_size,
|
|
quant_config=quant_config,
|
|
use_attn_tp_group=is_dp_attention_enabled(),
|
|
prefix=add_prefix("embed_tokens", prefix),
|
|
)
|
|
else:
|
|
self.embed_tokens = PPMissingLayer()
|
|
|
|
# Use the provided decoder layer type or default to MiMoV2DecoderLayer
|
|
decoder_layer_type = decoder_layer_type or MiMoV2DecoderLayer
|
|
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
|
config.num_hidden_layers,
|
|
layer_fn=lambda idx, prefix: decoder_layer_type(
|
|
layer_id=idx,
|
|
config=config,
|
|
quant_config=quant_config,
|
|
prefix=prefix,
|
|
),
|
|
prefix=add_prefix("layers", prefix),
|
|
stage_facts=partial(decoder_layer_type.stage_facts, config),
|
|
)
|
|
if self.pp_group.is_last_rank:
|
|
self.norm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
|
|
else:
|
|
self.norm = PPMissingLayer(return_tuple=True)
|
|
if uses_cutedsl_ar_fusion():
|
|
install_cutedsl_fusion(
|
|
self.layers[self.start_layer : self.end_layer],
|
|
hidden_size=config.hidden_size,
|
|
top_k=config.num_experts_per_tok,
|
|
rms_epsilon=config.layernorm_epsilon,
|
|
can_defer_finalize=lambda layer: (
|
|
isinstance(layer.mlp, MiMoV2MoE)
|
|
and layer.mlp.supports_deferred_finalize
|
|
),
|
|
label="MiMo",
|
|
include_shared_expert=False,
|
|
)
|
|
|
|
def get_input_embedding(self, input_ids: torch.Tensor) -> torch.Tensor:
|
|
if hasattr(self.config, "scale_emb"):
|
|
return self.get_input_embeddings()(input_ids) * self.config.scale_emb
|
|
else:
|
|
return self.get_input_embeddings()(input_ids)
|
|
|
|
def get_input_embeddings(self) -> nn.Embedding:
|
|
return self.embed_tokens
|
|
|
|
def forward(
|
|
self,
|
|
input_ids: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
input_embeds: torch.Tensor = None,
|
|
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
|
) -> Union[torch.Tensor, PPProxyTensors]:
|
|
if self.pp_group.is_first_rank:
|
|
if input_embeds is None:
|
|
hidden_states = self.embed_tokens(input_ids)
|
|
else:
|
|
hidden_states = input_embeds
|
|
residual_batch.start(forward_batch)
|
|
else:
|
|
assert pp_proxy_tensors is not None
|
|
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
|
pp_proxy_tensors, forward_batch
|
|
)
|
|
|
|
aux_hidden_states = AuxHiddenStatePacker(len(self.layers_to_capture))
|
|
if forward_batch.can_run_tbo and not self.layers_to_capture:
|
|
tbo_start_layer = self.start_layer
|
|
tbo_end_layer = self.end_layer
|
|
|
|
# skip first layer for TBO when starting from layer 0
|
|
if self.start_layer == 0:
|
|
layer = self.layers[0]
|
|
hidden_states = layer(positions, hidden_states, forward_batch)
|
|
tbo_start_layer = tbo_start_layer + 1
|
|
|
|
hidden_states = model_forward_stages(
|
|
layers=self.layers[tbo_start_layer:tbo_end_layer],
|
|
enable_tbo=True,
|
|
positions=positions,
|
|
forward_batch=forward_batch,
|
|
hidden_states=hidden_states,
|
|
)
|
|
else:
|
|
for i in range(self.start_layer, self.end_layer):
|
|
layer = self.layers[i]
|
|
hidden_states = layer(
|
|
positions,
|
|
hidden_states,
|
|
forward_batch,
|
|
captured_last_layer_outputs=aux_hidden_states
|
|
if i in self.layers_to_capture
|
|
else None,
|
|
)
|
|
|
|
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
|
|
|
|
# A draft targeting the final layer ("after layer
|
|
# num_hidden_layers-1") maps to capture index num_hidden_layers,
|
|
# past the layer loop; capture the pre-norm output here instead.
|
|
if (
|
|
self.pp_group.is_last_rank
|
|
and self.config.num_hidden_layers in self.layers_to_capture
|
|
):
|
|
aux_hidden_states.append(
|
|
residual_batch.snapshot(hidden_states, forward_batch)
|
|
)
|
|
|
|
hidden_states_before_norm = None
|
|
if not self.pp_group.is_last_rank:
|
|
return residual_batch.to_pp(hidden_states, forward_batch)
|
|
else:
|
|
if hidden_states.shape[0] > 0:
|
|
if forward_batch.return_hidden_states_before_norm:
|
|
hidden_states_before_norm = residual_batch.snapshot(
|
|
hidden_states, forward_batch
|
|
)
|
|
hidden_states = residual_batch.final_norm(
|
|
hidden_states, forward_batch, self.norm
|
|
)
|
|
|
|
if len(aux_hidden_states) == 0:
|
|
return hidden_states, hidden_states_before_norm
|
|
return hidden_states, hidden_states_before_norm, aux_hidden_states.finalize()
|
|
|
|
# If this function is called, it should always initialize KV cache scale
|
|
# factors (or else raise an exception). Thus, handled exceptions should
|
|
# make sure to leave KV cache scale factors in a known good (dummy) state
|
|
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
|
|
attn_tp_rank, attn_tp_size = self._kv_cache_parallel_layout
|
|
for layer_idx, scaling_factor in kv_cache_scales_loader(
|
|
quantization_param_path,
|
|
attn_tp_rank,
|
|
attn_tp_size,
|
|
self.config.num_hidden_layers,
|
|
self.config.__class__.model_type,
|
|
):
|
|
if not isinstance(self.layers[layer_idx], nn.Identity):
|
|
layer_self_attn = self.layers[layer_idx].self_attn
|
|
if hasattr(layer_self_attn.attn, "k_scale"):
|
|
layer_self_attn.attn.k_scale = scaling_factor
|
|
layer_self_attn.attn.v_scale = scaling_factor
|
|
else:
|
|
raise RuntimeError(
|
|
"Self attention has no KV cache scaling factor attribute!"
|
|
)
|
|
|
|
|
|
class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
|
|
# BitandBytes specific attributes
|
|
default_bitsandbytes_target_modules = [
|
|
".gate_proj.",
|
|
".down_proj.",
|
|
".up_proj.",
|
|
".q_proj.",
|
|
".k_proj.",
|
|
".v_proj.",
|
|
".o_proj.",
|
|
]
|
|
bitsandbytes_stacked_params_mapping = {
|
|
# shard_name, weight_name, index
|
|
"q_proj": ("qkv_proj", 0),
|
|
"k_proj": ("qkv_proj", 1),
|
|
"v_proj": ("qkv_proj", 2),
|
|
"gate_proj": ("gate_up_proj", 0),
|
|
"up_proj": ("gate_up_proj", 1),
|
|
}
|
|
|
|
# Prefixes for weight routing in encoder_only/language_only modes
|
|
_LANGUAGE_WEIGHT_PREFIXES = ("model.", "lm_head.")
|
|
_VISION_WEIGHT_PREFIXES = ("visual.", "vision_model.")
|
|
# ``audio_`` already covers ``audio_encoder.`` so a single prefix is enough.
|
|
_AUDIO_WEIGHT_PREFIXES = ("audio_",)
|
|
_AUDIO_WEIGHT_SUBSTRING = "speech_embeddings"
|
|
|
|
def __init__(
|
|
self,
|
|
config: MiMoV2Config,
|
|
quant_config: Optional[QuantizationConfig] = None,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
self.pp_group = get_parallel().pp_group
|
|
self.config = config
|
|
self.quant_config = quant_config
|
|
self._encoder_processor = None # lazy-created in preprocess_mm_for_encoder
|
|
|
|
if not self.config.encoder_only:
|
|
self.model = MiMoV2Model(
|
|
config, quant_config=quant_config, prefix=add_prefix("model", prefix)
|
|
)
|
|
|
|
if self.pp_group.is_last_rank:
|
|
self.lm_head = ParallelLMHead(
|
|
config.vocab_size,
|
|
config.hidden_size,
|
|
quant_config=quant_config,
|
|
prefix=add_prefix("lm_head", prefix),
|
|
use_attn_tp_group=get_parallel().enable_dp_lm_head,
|
|
)
|
|
else:
|
|
self.lm_head = PPMissingLayer()
|
|
else:
|
|
self.model = None
|
|
self.lm_head = None
|
|
|
|
self.logits_processor = (
|
|
LogitsProcessor(config) if not self.config.encoder_only else None
|
|
)
|
|
self.capture_aux_hidden_states = False
|
|
|
|
vision_config = getattr(config, "vision_config", None)
|
|
audio_config = getattr(config, "audio_config", None)
|
|
self._is_multimodal = vision_config is not None and audio_config is not None
|
|
# Always build vision/audio encoders so P can fall back to local
|
|
# encoding when the EPD encoder is unreachable.
|
|
if self._is_multimodal:
|
|
if hasattr(vision_config, "to_dict"):
|
|
vision_config = vision_config.to_dict()
|
|
if hasattr(audio_config, "to_dict"):
|
|
audio_config = audio_config.to_dict()
|
|
|
|
self.visual = MiMoVisionTransformer(
|
|
MiMoVLVisionConfig.from_dict(vision_config),
|
|
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
|
quant_config=None,
|
|
prefix=add_prefix("visual", prefix),
|
|
)
|
|
self.build_audio_encoder(MiMoAudioEncoderConfig(**audio_config))
|
|
|
|
self._routed_experts_weights_of_layer = LazyValue(
|
|
lambda: (
|
|
{
|
|
layer_id: layer.mlp.get_moe_weights()
|
|
for layer_id, layer in enumerate(self.model.layers)
|
|
if isinstance(layer.mlp, MiMoV2MoE)
|
|
}
|
|
if self.model is not None
|
|
else {}
|
|
)
|
|
)
|
|
|
|
@property
|
|
def routed_experts_weights_of_layer(self):
|
|
return self._routed_experts_weights_of_layer.value
|
|
|
|
def get_input_embedding(self, input_ids: torch.Tensor) -> torch.Tensor:
|
|
assert self.model is not None, (
|
|
"get_input_embedding() is not available in encoder_only mode"
|
|
)
|
|
return self.model.get_input_embedding(input_ids)
|
|
|
|
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
|
return pad_mm_tokens(input_ids, mm_inputs)
|
|
|
|
def preprocess_mm_for_encoder(self, mm_data, modality, config):
|
|
if self._encoder_processor is None:
|
|
from sglang.srt.multimodal.processors.mimo_v2 import MiMoProcessor
|
|
|
|
self._encoder_processor = MiMoProcessor.from_hf_config(
|
|
self.config, mm_config=config
|
|
)
|
|
return self._encoder_processor.preprocess_for_encoder(mm_data, modality)
|
|
|
|
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
|
pixel_values = torch.cat([item.feature for item in items], dim=0).type(
|
|
self.visual.dtype
|
|
)
|
|
image_grid_thw = torch.cat([item.image_grid_thw for item in items], dim=0)
|
|
assert pixel_values.dim() == 2, pixel_values.dim()
|
|
assert image_grid_thw.dim() == 2, image_grid_thw.dim()
|
|
return self.visual(pixel_values, grid_thw=image_grid_thw)
|
|
|
|
def get_video_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
|
pixel_values = torch.cat([item.feature for item in items], dim=0).type(
|
|
self.visual.dtype
|
|
)
|
|
video_grid_thw = torch.cat([item.video_grid_thw for item in items], dim=0)
|
|
assert pixel_values.dim() == 2, pixel_values.dim()
|
|
assert video_grid_thw.dim() == 2, video_grid_thw.dim()
|
|
return self.visual(pixel_values, grid_thw=video_grid_thw)
|
|
|
|
@torch.inference_mode()
|
|
def encode_video_audio(self, mm_inputs: Dict) -> Optional[torch.Tensor]:
|
|
# EPD-side hook: encode audio tracks pulled from videos and trim to the
|
|
# interleaved per-video segments produced by MiMoProcessor (segment
|
|
# starts / lens / per_video_num_units). Returns None if there is no
|
|
# audio to encode. The server passes the result through to the receiver
|
|
# under aux_data["video_audio_embedding"].
|
|
import numpy as np
|
|
|
|
audio_features = mm_inputs.get("video_audio_features")
|
|
if not audio_features:
|
|
return None
|
|
|
|
def _as_tensor(data):
|
|
if isinstance(data, torch.Tensor):
|
|
return data
|
|
if isinstance(data, np.ndarray):
|
|
return torch.tensor(data)
|
|
if isinstance(data, list) and data and isinstance(data[0], np.ndarray):
|
|
return torch.tensor(np.array(data))
|
|
if isinstance(data, list) and data and isinstance(data[0], (int, float)):
|
|
return torch.tensor(data)
|
|
return data
|
|
|
|
audio_feature_lens = mm_inputs["video_audio_feature_lens"]
|
|
audio_item = MultimodalDataItem.from_dict(
|
|
{
|
|
"modality": Modality.AUDIO,
|
|
"feature": _as_tensor(audio_features),
|
|
}
|
|
)
|
|
audio_item.set("audio_feature_lens", _as_tensor(audio_feature_lens))
|
|
|
|
audio_embedding = self.get_audio_feature([audio_item]).cpu()
|
|
if audio_embedding.ndim != 2:
|
|
audio_embedding = audio_embedding.reshape(-1, audio_embedding.shape[-1])
|
|
|
|
segment_lens_flat = mm_inputs["video_audio_segment_lens_flat"]
|
|
segment_starts_flat = mm_inputs["video_audio_segment_starts_flat"]
|
|
per_video_num_units = mm_inputs["video_audio_per_video_num_units"]
|
|
per_video_audio_token_lens = (
|
|
audio_feature_lens.tolist()
|
|
if hasattr(audio_feature_lens, "tolist")
|
|
else list(audio_feature_lens)
|
|
)
|
|
|
|
trimmed_chunks = []
|
|
emb_offset = 0
|
|
unit_idx = 0
|
|
audio_video_idx = 0
|
|
for num_units in per_video_num_units:
|
|
if num_units <= 0:
|
|
continue
|
|
vid_audio_len = per_video_audio_token_lens[audio_video_idx]
|
|
for _ in range(num_units):
|
|
start = segment_starts_flat[unit_idx]
|
|
seg_len = segment_lens_flat[unit_idx]
|
|
trimmed_chunks.append(
|
|
audio_embedding[emb_offset + start : emb_offset + start + seg_len]
|
|
)
|
|
unit_idx += 1
|
|
emb_offset += vid_audio_len
|
|
audio_video_idx += 1
|
|
|
|
return (
|
|
torch.cat(trimmed_chunks, dim=0) if trimmed_chunks else audio_embedding[:0]
|
|
)
|
|
|
|
def get_input_embeddings(self) -> Optional[nn.Embedding]:
|
|
return self.model.embed_tokens if self.model is not None else None
|
|
|
|
@torch.no_grad()
|
|
def forward(
|
|
self,
|
|
input_ids: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
forward_batch: ForwardBatch,
|
|
input_embeds: torch.Tensor = None,
|
|
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
|
) -> torch.Tensor:
|
|
assert not self.config.encoder_only, (
|
|
"forward() should not be called in encoder_only mode"
|
|
)
|
|
|
|
aux_hidden_states = None
|
|
# Multimodal embedding must run even when aux hidden states are captured
|
|
# (DFLASH/MTP). Taking the capture branch first skipped image/audio
|
|
# feature placement and left placeholder tokens as plain text embeddings.
|
|
if self._is_multimodal:
|
|
model_out = general_mm_embed_routine(
|
|
input_ids=input_ids,
|
|
forward_batch=forward_batch,
|
|
language_model=self.model,
|
|
multimodal_model=self,
|
|
positions=positions,
|
|
pp_proxy_tensors=pp_proxy_tensors,
|
|
)
|
|
else:
|
|
model_out = self.model(
|
|
input_ids,
|
|
positions,
|
|
forward_batch,
|
|
input_embeds,
|
|
pp_proxy_tensors=pp_proxy_tensors,
|
|
)
|
|
if self.capture_aux_hidden_states:
|
|
hidden_states, hidden_states_before_norm, aux_hidden_states = model_out
|
|
else:
|
|
hidden_states, hidden_states_before_norm = model_out
|
|
|
|
if self.pp_group.is_last_rank:
|
|
return self.logits_processor(
|
|
input_ids,
|
|
hidden_states,
|
|
self.lm_head,
|
|
forward_batch,
|
|
hidden_states_before_norm=hidden_states_before_norm,
|
|
aux_hidden_states=aux_hidden_states,
|
|
)
|
|
else:
|
|
return hidden_states
|
|
|
|
@property
|
|
def start_layer(self):
|
|
return self.model.start_layer if self.model is not None else 0
|
|
|
|
@property
|
|
def end_layer(self):
|
|
return self.model.end_layer if self.model is not None else 0
|
|
|
|
def set_dflash_layers_to_capture(self, layer_ids: List[int]):
|
|
if not self.pp_group.is_last_rank:
|
|
return
|
|
|
|
if layer_ids is None:
|
|
raise ValueError(
|
|
"DFLASH requires explicit layer_ids for aux hidden capture."
|
|
)
|
|
|
|
self.capture_aux_hidden_states = True
|
|
# target_layer_ids are "after layer X" ids; capture before layer X+1,
|
|
# matching the draft's extract_context_feature (offset=1).
|
|
self.model.layers_to_capture = [val + 1 for val in layer_ids]
|
|
|
|
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
|
stacked_params_mapping = [
|
|
# (param_name, shard_name, shard_id)
|
|
("qkv_proj", "q_proj", "q"),
|
|
("qkv_proj", "k_proj", "k"),
|
|
("qkv_proj", "v_proj", "v"),
|
|
("gate_up_proj", "gate_proj", 0),
|
|
("gate_up_proj", "up_proj", 1),
|
|
]
|
|
stacked_params_mapping_vit = [
|
|
# (param_name, shard_name, shard_id)
|
|
("gate_up_proj", "gate_proj", 0),
|
|
("gate_up_proj", "up_proj", 1),
|
|
]
|
|
|
|
# (param_name, weight_name, expert_id, shard_id)
|
|
expert_params_mapping = DeepEPMoE.make_expert_params_mapping(
|
|
ckpt_gate_proj_name="gate_proj",
|
|
ckpt_down_proj_name="down_proj",
|
|
ckpt_up_proj_name="up_proj",
|
|
num_experts=self.config.n_routed_experts,
|
|
)
|
|
|
|
params_dict = dict(self.named_parameters())
|
|
skipped_mtp_weights = False
|
|
deferred_qkv_scale_inv: Dict[str, torch.Tensor] = {}
|
|
|
|
for name, loaded_weight in weights:
|
|
is_vision_weight = name.startswith(self._VISION_WEIGHT_PREFIXES)
|
|
is_audio_weight = (
|
|
name.startswith(self._AUDIO_WEIGHT_PREFIXES)
|
|
or self._AUDIO_WEIGHT_SUBSTRING in name
|
|
)
|
|
|
|
if not self._is_multimodal and (is_vision_weight or is_audio_weight):
|
|
continue
|
|
|
|
if self.config.encoder_only and name.startswith(
|
|
self._LANGUAGE_WEIGHT_PREFIXES
|
|
):
|
|
continue
|
|
|
|
if self._is_multimodal and is_audio_weight:
|
|
if name.startswith("audio_encoder."):
|
|
name = name[len("audio_encoder.") :]
|
|
name = self.remap_audio_weight_name(name)
|
|
if "input_local_transformer" not in name:
|
|
name = name.replace("self_attn.out_proj", "self_attn.attn.proj")
|
|
audio_stacked = False
|
|
for param_name, weight_name, shard_id in [
|
|
("self_attn.attn.qkv_proj", "self_attn.q_proj", "q"),
|
|
("self_attn.attn.qkv_proj", "self_attn.k_proj", "k"),
|
|
("self_attn.attn.qkv_proj", "self_attn.v_proj", "v"),
|
|
]:
|
|
if weight_name in name:
|
|
name = name.replace(weight_name, param_name)
|
|
if name not in params_dict:
|
|
break
|
|
param = params_dict[name]
|
|
weight_loader = param.weight_loader
|
|
weight_loader(param, loaded_weight, shard_id)
|
|
audio_stacked = True
|
|
break
|
|
if audio_stacked:
|
|
continue
|
|
if name not in params_dict:
|
|
logger.warning(
|
|
f"Audio param {name} not found in params_dict, skipping"
|
|
)
|
|
continue
|
|
param = params_dict[name]
|
|
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
|
if self._AUDIO_WEIGHT_SUBSTRING in name:
|
|
weight_loader(param, loaded_weight[: param.shape[0], :])
|
|
else:
|
|
weight_loader(param, loaded_weight)
|
|
continue
|
|
|
|
if self._is_multimodal and "visual" in name:
|
|
name = name.replace("vision_model.", "")
|
|
name = name.replace(r"attn.qkv.", r"attn.qkv_proj.")
|
|
match_stacked_vit = False
|
|
for param_name, weight_name, shard_id in stacked_params_mapping_vit:
|
|
if weight_name not in name:
|
|
continue
|
|
name = name.replace(weight_name, param_name)
|
|
# Skip loading extra bias for GPTQ models.
|
|
if name.endswith(".bias") and name not in params_dict:
|
|
match_stacked_vit = True
|
|
continue
|
|
param = params_dict[name]
|
|
weight_loader = param.weight_loader
|
|
weight_loader(param, loaded_weight, shard_id)
|
|
match_stacked_vit = True
|
|
break
|
|
if match_stacked_vit:
|
|
continue
|
|
# Skip loading extra bias for GPTQ models.
|
|
if name.endswith(".bias") and name not in params_dict:
|
|
continue
|
|
param = params_dict[name]
|
|
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
|
weight_loader(param, loaded_weight)
|
|
|
|
if name.endswith("patch_embed.proj.weight"):
|
|
patch_embed = self.get_submodule(name.rsplit(".", 2)[0])
|
|
if hasattr(patch_embed, "sync_proj_weight_linear_format"):
|
|
patch_embed.sync_proj_weight_linear_format()
|
|
continue
|
|
|
|
layer_id = get_layer_id(name)
|
|
if (
|
|
layer_id is not None
|
|
and hasattr(self.model, "start_layer")
|
|
and (
|
|
layer_id < self.model.start_layer
|
|
or layer_id >= self.model.end_layer
|
|
)
|
|
):
|
|
continue
|
|
|
|
if "rotary_emb.inv_freq" in name or "projector" in name:
|
|
continue
|
|
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
|
|
# Models trained using ColossalAI may include these tensors in
|
|
# the checkpoint. Skip them.
|
|
continue
|
|
|
|
if self.config.tie_word_embeddings and "lm_head.weight" in name:
|
|
if self.pp_group.world_size > 1 and self.pp_group.is_last_rank:
|
|
# Handle pp weight tying here
|
|
# find the embed_tokens.weight in the weights
|
|
embed_token_weights = next(
|
|
filter(lambda x: x[0] == "model.embed_tokens.weight", weights)
|
|
)[1]
|
|
loaded_weight = embed_token_weights
|
|
else:
|
|
continue
|
|
|
|
if "mtp" in name:
|
|
if not skipped_mtp_weights:
|
|
logger.info(
|
|
"Skipping draft-only MiMo-V2 MTP weights while loading the "
|
|
"target model; MiMoV2MTP loads these weights in the draft "
|
|
"model runner."
|
|
)
|
|
skipped_mtp_weights = True
|
|
continue
|
|
|
|
if ".mlp.experts." in name and loaded_weight.dtype == torch.uint8:
|
|
if name.endswith(".weight_scale"):
|
|
name = name + "_inv"
|
|
loaded_weight = torch.exp2(loaded_weight.to(torch.float32) - 127.0)
|
|
elif name.endswith(".weight"):
|
|
loaded_weight = loaded_weight.view(torch.int8)
|
|
|
|
# Support fused qkv_proj checkpoint (Pro format)
|
|
if "qkv_proj" in name:
|
|
if name in params_dict:
|
|
param = params_dict[name]
|
|
expected_fused_tp_size = get_mimo_v2_fused_qkv_expected_tp_size(
|
|
self.config
|
|
)
|
|
load_mimo_v2_qkv_proj_weight(
|
|
name,
|
|
param,
|
|
loaded_weight,
|
|
expected_fused_tp_size,
|
|
deferred_scale_inv=deferred_qkv_scale_inv,
|
|
qkv_proj=self.get_submodule(name.rsplit(".", 1)[0]),
|
|
)
|
|
continue
|
|
|
|
for param_name, weight_name, shard_id in stacked_params_mapping:
|
|
if (
|
|
"compression_attention" in name
|
|
or "hybrid_softmax_attention" in name
|
|
or "compressed_softmax_attn" in name
|
|
):
|
|
continue
|
|
if weight_name not in name:
|
|
continue
|
|
if ("mlp.experts." in name) and name not in params_dict:
|
|
continue
|
|
|
|
name = name.replace(weight_name, param_name)
|
|
# Skip loading extra bias for GPTQ models.
|
|
if name.endswith(".bias") and name not in params_dict:
|
|
continue
|
|
|
|
param = params_dict[name]
|
|
weight_loader = param.weight_loader
|
|
weight_loader(param, loaded_weight, shard_id)
|
|
break
|
|
else:
|
|
for mapping in expert_params_mapping:
|
|
param_name, weight_name, expert_id, shard_id = mapping
|
|
if weight_name not in name:
|
|
continue
|
|
name = name.replace(weight_name, param_name)
|
|
# mxfp4 ckpts store expert scales without the `_inv` suffix,
|
|
# while Fp8MoEMethod registers them as *_weight_scale_inv.
|
|
if name.endswith("weight_scale") and (name + "_inv" in params_dict):
|
|
name = name + "_inv"
|
|
param = params_dict[name]
|
|
weight_loader = param.weight_loader
|
|
weight_loader(
|
|
param,
|
|
loaded_weight,
|
|
name,
|
|
shard_id=shard_id,
|
|
expert_id=expert_id,
|
|
)
|
|
break
|
|
else:
|
|
# Skip loading extra bias for GPTQ models.
|
|
if name.endswith(".bias") and name not in params_dict:
|
|
continue
|
|
|
|
if name in params_dict.keys():
|
|
param = params_dict[name]
|
|
if "attention_sink_bias" in name:
|
|
projection = unwrap_lora_layer(
|
|
self.get_submodule(name.rsplit(".", 1)[0]).qkv_proj
|
|
)
|
|
start = (
|
|
get_group_rank_size(projection.tp_group)[0]
|
|
* param.numel()
|
|
)
|
|
param.data.copy_(
|
|
loaded_weight[start : start + param.numel()]
|
|
)
|
|
else:
|
|
weight_loader = getattr(
|
|
param, "weight_loader", default_weight_loader
|
|
)
|
|
weight_loader(param, loaded_weight)
|
|
else:
|
|
logger.warning(f"Parameter {name} not found in params_dict")
|
|
|
|
if deferred_qkv_scale_inv:
|
|
expected_fused_tp_size = get_mimo_v2_fused_qkv_expected_tp_size(self.config)
|
|
_resolve_deferred_qkv_scale_inv(
|
|
params_dict,
|
|
deferred_qkv_scale_inv,
|
|
expected_fused_tp_size,
|
|
config=self.config,
|
|
model=self,
|
|
)
|
|
|
|
def get_embed_and_head(self):
|
|
assert self.model is not None and self.lm_head is not None, (
|
|
"get_embed_and_head() is not available in encoder_only mode"
|
|
)
|
|
return self.model.embed_tokens.weight, self.lm_head.weight
|
|
|
|
def set_embed_and_head(self, embed, head):
|
|
assert self.model is not None and self.lm_head is not None, (
|
|
"set_embed_and_head() is not available in encoder_only mode"
|
|
)
|
|
del self.model.embed_tokens.weight
|
|
del self.lm_head.weight
|
|
self.model.embed_tokens.weight = embed
|
|
self.lm_head.weight = head
|
|
torch.cuda.empty_cache()
|
|
torch.cuda.synchronize()
|
|
|
|
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
|
|
if self.model is not None:
|
|
self.model.load_kv_cache_scales(quantization_param_path)
|
|
|
|
@classmethod
|
|
def get_model_config_for_expert_location(cls, config):
|
|
return ModelConfigForExpertLocation(
|
|
num_layers=config.num_hidden_layers,
|
|
num_logical_experts=getattr(config, "n_routed_experts", 1),
|
|
num_groups=getattr(config, "n_group", None),
|
|
)
|
|
|
|
|
|
# Keep the old Flash architecture name loadable while new configs use MiMoV2ForCausalLM.
|
|
class MiMoV2FlashForCausalLM(MiMoV2ForCausalLM):
|
|
pass
|
|
|
|
|
|
EntryClass = [MiMoV2ForCausalLM, MiMoV2FlashForCausalLM]
|