Files
sglang/python/sglang/srt/models/dots3_common/modeling.py
T

2576 lines
99 KiB
Python

# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Adapted from vLLM's DeepSeek V2 implementation:
# https://github.com/vllm-project/vllm/blob/fb6af8bc086328ca6659e72d11ffd4309ce4de22/vllm/model_executor/models/deepseek_v2.py
"""Inference implementation for dots.note.omni."""
import concurrent.futures
import logging
import math
from array import array
from dataclasses import dataclass
from enum import IntEnum, auto
from typing import (
Any,
Callable,
Iterable,
List,
Optional,
Protocol,
Tuple,
Union,
runtime_checkable,
)
import torch
import torch.nn.functional as F
from torch import nn
from transformers import PretrainedConfig
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.batch_overlap.two_batch_overlap import (
MaybeTboDeepEPDispatcher,
model_forward_stages,
)
from sglang.srt.configs.dots3 import Dots3Config
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_deepep_mode,
get_moe_a2a_backend,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8_utils import (
block_quant_dequant,
requant_weight_ue8m0_inplace,
)
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope_wrapper
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.managers.mm_utils import (
general_mm_embed_routine,
pad_mm_token_pairs,
)
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalInputs,
)
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_executor.forward_context import (
get_attn_backend,
get_token_to_kv_pool,
)
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
_load_fused_indexer_wk,
)
from sglang.srt.models.deepseek_common.utils import tiny_router_gemm_max_tokens
from sglang.srt.models.dots3_common.fp8 import per_token_group_quant_einsum_fp8
from sglang.srt.runtime_context import (
get_device,
get_exec,
get_parallel,
)
from sglang.srt.utils import (
BumpAllocator,
LazyValue,
add_prefix,
bind_or_assign,
ceil_align,
ceil_div,
is_cuda,
log_info_on_rank0,
make_pp_layers,
)
_is_cuda = is_cuda()
_is_fp8_fnuz = is_fp8_fnuz()
# Import-time CUDA kernels would block processor imports on CPU CI.
if _is_cuda:
from sgl_kernel import merge_state_v2
from sglang.kernels.ops.gemm.tiny_gemm import tiny_gemm_bf16
else:
merge_state_v2 = None
tiny_gemm_bf16 = None
def _require_cuda() -> None:
if not _is_cuda:
raise RuntimeError("Dots3 model only supports CUDA backend.")
logger = logging.getLogger(__name__)
@runtime_checkable
class _SupportsWeightLoader(Protocol):
weight_loader: Callable[..., Any]
def _get_scale_block_n(
quant_config: Optional[QuantizationConfig],
) -> int:
if quant_config is not None and quant_config.weight_block_size is not None:
return quant_config.weight_block_size[0]
return 1
def _get_param_weight_loader(param: nn.Parameter):
return (
param.weight_loader
if isinstance(param, _SupportsWeightLoader)
else default_weight_loader
)
class Dots3AttnForwardMethod(IntEnum):
# Use absorbed multi-latent attention
MLA = auto()
# Use multi-head attention, but with KV cache chunked.
# This method can avoid OOM when prefix lengths are long.
MHA_CHUNKED_KV = auto()
# Use dense MHA for short DSA prefills selected by the DSA backend.
SWA_MHA = auto()
class Dots3MLP(nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
reduce_results: bool = True,
prefix: str = "",
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.is_replicated = parallel_group == "replicated"
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
parallel_group=parallel_group,
)
if hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
)
self.act_fn = SiluAndMul()
def forward(self, x, forward_batch=None):
if self.is_replicated and x.shape[0] == 0:
return x
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
class Dots3MoEGate(nn.Module):
def __init__(self, config):
super().__init__()
self.weight = nn.Parameter(
torch.empty((config.n_routed_experts, config.hidden_size))
)
if config.topk_method == "noaux_tc":
self.e_score_correction_bias = nn.Parameter(
torch.empty((config.n_routed_experts), dtype=torch.float32)
)
else:
self.e_score_correction_bias = None
self.tiny_router_gemm_max_tokens = tiny_router_gemm_max_tokens(
num_experts=config.n_routed_experts,
hidden_size=config.hidden_size,
weight_dtype=self.weight.dtype,
)
def forward(self, hidden_states):
# Use the fused router only for its tuned shapes.
if hidden_states.shape[0] <= self.tiny_router_gemm_max_tokens:
logits = tiny_gemm_bf16(
hidden_states, self.weight, max_m=self.tiny_router_gemm_max_tokens
)
else:
logits = F.linear(hidden_states, self.weight, None)
return logits
class Dots3MoE(nn.Module):
def __init__(
self,
config: PretrainedConfig,
layer_id: int,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
alt_stream: Optional[torch.cuda.Stream] = None,
is_nextn: bool = False,
):
super().__init__()
self.tp_size = get_parallel().tp_size
self.routed_scaling_factor = config.routed_scaling_factor
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.config = config
self.layer_id = layer_id
self.alt_stream = alt_stream
if self.tp_size > config.n_routed_experts:
raise ValueError(
f"Tensor parallel size {self.tp_size} is greater than "
f"the number of experts {config.n_routed_experts}."
)
if config.hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {config.hidden_act}. "
"Only silu is supported for now."
)
self.gate = Dots3MoEGate(config)
self.experts = get_moe_impl_class(quant_config)(
num_experts=config.n_routed_experts
+ self.num_fused_shared_experts
+ get_exec().moe.ep_num_redundant_experts,
num_fused_shared_experts=self.num_fused_shared_experts,
top_k=config.num_experts_per_tok + self.num_fused_shared_experts,
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
layer_id=self.layer_id,
quant_config=quant_config,
routed_scaling_factor=self.routed_scaling_factor,
prefix=add_prefix("experts", prefix),
)
self.topk = TopK(
top_k=config.num_experts_per_tok,
layer_id=self.layer_id,
renormalize=config.norm_topk_prob,
use_grouped_topk=True,
num_expert_group=config.n_group,
topk_group=config.topk_group,
correction_bias=self.gate.e_score_correction_bias,
routed_scaling_factor=self.routed_scaling_factor,
apply_routed_scaling_factor_on_output=self.experts.should_fuse_routed_scaling_factor_in_topk,
)
self.shared_experts = None
if config.n_shared_experts is not None and self.num_fused_shared_experts == 0:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
# disable tp for shared experts when enable deepep moe, or with fp4 allgather
self.shared_experts = Dots3MLP(
hidden_size=config.hidden_size,
intermediate_size=intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("shared_experts", prefix),
parallel_group="replicated"
if get_moe_a2a_backend().is_deepep()
or should_use_flashinfer_cutlass_moe_fp4_allgather()
else "tp",
)
is_packed_weight = quant_config is not None and (
quant_config.get_name()
in {
"awq",
"awq_marlin",
"moe_wna16",
}
)
shared_experts_is_fp8 = (
not is_packed_weight
and self.shared_experts.gate_up_proj.weight.dtype == torch.float8_e4m3fn
)
if shared_experts_is_fp8:
assert (
self.shared_experts.gate_up_proj.quant_method.quant_config.weight_block_size
== self.shared_experts.down_proj.quant_method.quant_config.weight_block_size
)
self.top_k = config.num_experts_per_tok
if get_moe_a2a_backend().is_deepep():
# TODO: we will support tp < ep in the future
self.ep_size = get_parallel().moe_ep_size
self.num_experts = (
config.n_routed_experts + get_exec().moe.ep_num_redundant_experts
)
self.renormalize = config.norm_topk_prob
self.topk_group = config.topk_group
self.num_expert_group = config.n_group
self.correction_bias = (
self.gate.e_score_correction_bias.data
if self.gate.e_score_correction_bias is not None
else None
)
self.deepep_dispatcher = MaybeTboDeepEPDispatcher(
group=get_parallel().tp_group.device_group,
router_topk=self.top_k,
permute_fusion=True,
num_experts=self.num_experts,
num_local_experts=config.n_routed_experts // self.tp_size,
hidden_size=config.hidden_size,
params_dtype=config.torch_dtype,
deepep_mode=get_deepep_mode(),
async_finish=True,
return_recv_hook=True,
)
self._enable_deepep_moe = get_moe_a2a_backend().is_deepep()
def get_moe_weights(self):
return [
x.data
for name, x in self.experts.named_parameters()
if name not in ["correction_bias"]
]
def forward(
self,
hidden_states: torch.Tensor,
forward_batch: Optional[ForwardBatch] = None,
) -> torch.Tensor:
if not self._enable_deepep_moe:
DUAL_STREAM_TOKEN_THRESHOLD = 1024
if (
self.alt_stream is not None
and self.num_fused_shared_experts == 0
and hidden_states.shape[0] > 0
and hidden_states.shape[0] <= DUAL_STREAM_TOKEN_THRESHOLD
):
return self.forward_normal_dual_stream(hidden_states)
else:
return self.forward_normal(hidden_states)
else:
return self.forward_deepep(hidden_states, forward_batch)
def forward_normal_dual_stream(self, hidden_states: torch.Tensor) -> torch.Tensor:
current_stream = torch.cuda.current_stream()
self.alt_stream.wait_stream(current_stream)
shared_output = self._forward_shared_experts(hidden_states)
with torch.cuda.stream(self.alt_stream):
# router_logits: (num_tokens, n_experts)
router_logits = self.gate(hidden_states)
topk_output = self.topk(hidden_states, router_logits)
final_hidden_states = self.experts(hidden_states, topk_output)
current_stream.wait_stream(self.alt_stream)
with use_symmetric_memory(get_parallel().tp_group) as sm:
final_hidden_states_out = torch.empty_like(final_hidden_states)
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
final_hidden_states = final_hidden_states_out
sm.tag(final_hidden_states)
return final_hidden_states
def forward_normal(self, hidden_states: torch.Tensor) -> torch.Tensor:
if hidden_states.shape[0] > 0:
shared_output = self._forward_shared_experts(hidden_states)
# router_logits: (num_tokens, n_experts)
router_logits = self.gate(hidden_states)
topk_output = self.topk(hidden_states, router_logits)
else:
shared_output = None
topk_output = self.topk.empty_topk_output(hidden_states.device)
final_hidden_states = self.experts(hidden_states, topk_output)
if shared_output is not None:
with use_symmetric_memory(get_parallel().tp_group) as sm:
final_hidden_states_out = torch.empty_like(final_hidden_states)
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
final_hidden_states = final_hidden_states_out
sm.tag(final_hidden_states)
return final_hidden_states
def forward_deepep(
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
shared_output = None
if hidden_states.shape[0] > 0:
# router_logits: (num_tokens, n_experts)
router_logits = self.gate(hidden_states)
shared_output = self._forward_shared_experts(hidden_states)
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
)
final_hidden_states = self.experts(
hidden_states=hidden_states,
topk_output=topk_output,
)
fused_in_topk = is_routed_scaling_factor_fused_in_topk(self.topk.topk_config)
if shared_output is not None:
x = shared_output
if fused_in_topk:
x.add_(final_hidden_states)
else:
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
final_hidden_states = x
elif not fused_in_topk:
final_hidden_states *= self.routed_scaling_factor
return final_hidden_states
def _forward_shared_experts(self, hidden_states):
if self.num_fused_shared_experts == 0:
return self.shared_experts(hidden_states)
else:
return None
# Aligned with HF's implementation, using sliding window inclusive with the last token.
# SGLang assumes exclusive.
def get_attention_sliding_window_size(config):
return config.sliding_window_size - 1
class Dots3AttentionMLA(nn.Module):
@dataclass
class Dots3AttentionMLAConfig:
attention_gate_type: str
kv_lora_rank: int
q_lora_rank: int
qk_nope_head_dim: int
qk_rope_head_dim: int
num_attention_heads: int
num_key_value_heads: int
v_head_dim: int
rope_theta: float
@classmethod
def from_config(
cls,
config: Dots3Config,
layer_type: str,
):
if layer_type == "sliding_attention":
return cls(
attention_gate_type=config.swa_attention_gate_type,
kv_lora_rank=config.swa_kv_lora_rank,
q_lora_rank=config.swa_q_lora_rank,
qk_nope_head_dim=config.swa_qk_nope_head_dim,
qk_rope_head_dim=config.swa_qk_rope_head_dim,
num_attention_heads=config.swa_num_attention_heads,
num_key_value_heads=config.swa_num_key_value_heads,
v_head_dim=config.swa_v_head_dim,
rope_theta=config.swa_rope_theta,
)
else:
return cls(
attention_gate_type=config.attention_gate_type,
kv_lora_rank=config.kv_lora_rank,
q_lora_rank=config.q_lora_rank,
qk_nope_head_dim=config.qk_nope_head_dim,
qk_rope_head_dim=config.qk_rope_head_dim,
num_attention_heads=config.num_attention_heads,
num_key_value_heads=config.num_key_value_heads,
v_head_dim=config.v_head_dim,
rope_theta=config.rope_theta,
)
def __init__(
self,
config: Dots3Config,
quant_config: Optional[QuantizationConfig] = None,
reduce_results: bool = False,
layer_id: int = None,
prefix: str = "",
alt_stream: Optional[torch.cuda.Stream] = None,
) -> None:
super().__init__()
attn_tp_size = get_parallel().attn_tp_size
self.layer_id = layer_id
self.hidden_size = config.hidden_size
# Determine the layer type and sliding window size
layer_type = config.layer_types[layer_id]
assert layer_type in {"sliding_attention", "full_attention"}
use_sliding_window = layer_type == "sliding_attention"
self.use_swa = use_sliding_window
self.sliding_window_size = (
get_attention_sliding_window_size(config) if use_sliding_window else -1
)
# Get Attention config based on layer type.
attn_config = self.Dots3AttentionMLAConfig.from_config(config, layer_type)
self.qk_nope_head_dim = attn_config.qk_nope_head_dim
self.qk_rope_head_dim = attn_config.qk_rope_head_dim
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
self.v_head_dim = attn_config.v_head_dim
self.q_lora_rank = attn_config.q_lora_rank
self.kv_lora_rank = attn_config.kv_lora_rank
self.apply_mla_qkv_lora_rescale = config.apply_mla_qkv_lora_rescale
self.num_heads = attn_config.num_attention_heads
assert self.num_heads % attn_tp_size == 0
self.num_local_heads = self.num_heads // attn_tp_size
assert attn_config.num_attention_heads == attn_config.num_key_value_heads, (
"Dots3 Only supports equal number of query and key value heads."
)
self.attention_gate_type = attn_config.attention_gate_type
assert self.attention_gate_type in {
"headwise",
"elementwise",
}, (
f"Unsupported attention_gate_type: {self.attention_gate_type}. Expected 'headwise' or 'elementwise'."
)
self.g_proj_local_dim = self.num_local_heads * (
1 if self.attention_gate_type == "headwise" else self.v_head_dim
)
self.scaling = self.qk_head_dim**-0.5
self.rope_theta = attn_config.rope_theta
self.max_position_embeddings = config.max_position_embeddings
# For tensor parallel attention
assert self.q_lora_rank is not None, "Dots3AttentionMLA requires q_lora_rank."
scale_block_n = _get_scale_block_n(quant_config)
self.qk_rope_head_dim_padded = ceil_align(self.qk_rope_head_dim, scale_block_n)
# NOTE(xiaozhi): For the sake of DeepGEMM kernel, we align the g_proj_local_dim to 8.
self.g_proj_local_dim_padded = ceil_align(
self.g_proj_local_dim, max(8, scale_block_n)
)
self.use_nsa = (
config.index_n_heads is not None
and config.index_head_dim is not None
and config.index_topk is not None
and layer_type == "full_attention"
)
fused_qkv_out_size = (
self.q_lora_rank
+ self.kv_lora_rank
+ self.qk_rope_head_dim_padded
+ self.g_proj_local_dim_padded
)
self.fused_qkv_a_g_proj_with_mqa = ReplicatedLinear(
self.hidden_size,
fused_qkv_out_size,
bias=False,
quant_config=quant_config,
prefix=add_prefix("fused_qkv_a_g_proj_with_mqa", prefix),
)
self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps)
self.q_b_out_size = self.num_heads * self.qk_head_dim
self.q_b_proj = ColumnParallelLinear(
self.q_lora_rank,
self.q_b_out_size,
bias=False,
quant_config=quant_config,
prefix=add_prefix("q_b_proj", prefix),
parallel_group="attn_tp",
)
self.kv_b_proj = ColumnParallelLinear(
self.kv_lora_rank,
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
bias=False,
quant_config=quant_config,
prefix=add_prefix("kv_b_proj", prefix),
parallel_group="attn_tp",
)
# O projection.
self.o_proj = RowParallelLinear(
self.num_heads * self.v_head_dim,
self.hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("o_proj", prefix),
parallel_group="attn_tp",
)
self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=config.rms_norm_eps)
self.k_rope_only_layernorm = RMSNorm(
self.qk_rope_head_dim, eps=config.rms_norm_eps
)
# Transformers v5 represents the default RoPE configuration as a
# non-empty ``{"rope_type": "default", ...}`` dictionary. Pass it
# through to the shared RoPE factory, which handles both that form and
# actual scaled-RoPE configurations.
self.rope_scaling = config.rope_scaling
self.rotary_emb = get_rope_wrapper(
head_size=self.qk_rope_head_dim,
rotary_dim=self.qk_rope_head_dim,
max_position=self.max_position_embeddings,
base=self.rope_theta,
rope_scaling=self.rope_scaling,
is_neox_style=False,
device=get_device().device,
)
# Optional NSA (Native Sparse Attention) indexer.
if self.use_nsa:
assert self.q_lora_rank is not None, (
"Dots3 NSA requires q_lora_rank to be set in the config."
)
self.indexer = Indexer(
hidden_size=self.hidden_size,
index_n_heads=config.index_n_heads,
index_head_dim=config.index_head_dim,
rope_head_dim=self.qk_rope_head_dim,
index_topk=config.index_topk,
q_lora_rank=self.q_lora_rank,
max_position_embeddings=self.max_position_embeddings,
rope_theta=self.rope_theta,
scale_fmt="ue8m0",
block_size=128,
rope_scaling=self.rope_scaling,
is_neox_style=False,
prefix=add_prefix("indexer", prefix),
quant_config=quant_config,
layer_id=self.layer_id,
alt_stream=alt_stream,
)
self.attn_mqa = RadixAttention(
num_heads=self.num_local_heads,
head_dim=self.kv_lora_rank + self.qk_rope_head_dim,
scaling=self.scaling,
num_kv_heads=1,
layer_id=self.layer_id,
v_head_dim=self.kv_lora_rank,
quant_config=quant_config,
prefix=add_prefix("attn_mqa", prefix),
sliding_window_size=self.sliding_window_size,
)
self.attn_mha = RadixAttention(
num_heads=self.num_local_heads,
head_dim=self.qk_head_dim,
scaling=self.scaling,
num_kv_heads=self.num_local_heads,
layer_id=self.layer_id,
v_head_dim=self.v_head_dim,
quant_config=quant_config,
prefix=add_prefix("attn_mha", prefix),
sliding_window_size=self.sliding_window_size,
)
self.alt_stream = alt_stream
self.w_kc = None
self.w_vc = None
self.w_scale_k = None
self.w_scale_v = None
# Attention backend used by current forward batch
self.current_attention_backend = None
def _split_fused_qkv_a_g_proj_out(
self, x: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
q_offset = 0
kv_offset = q_offset + self.q_lora_rank
g_offset = kv_offset + self.kv_lora_rank + self.qk_rope_head_dim_padded
q = x[..., q_offset : q_offset + self.q_lora_rank]
latent_cache = x[
..., kv_offset : kv_offset + self.kv_lora_rank + self.qk_rope_head_dim
]
g = x[..., g_offset : g_offset + self.g_proj_local_dim]
return q, latent_cache, g
def dispatch_attn_forward_method(
self, forward_batch: ForwardBatch
) -> Dots3AttnForwardMethod:
if (
self.use_swa
and forward_batch.forward_mode.is_context_parallel_extend()
and forward_batch.attn_cp_metadata is not None
):
raise NotImplementedError(
"Dots3 SWA attention does not support context-parallel prefill yet. "
"Please disable --enable-prefill-cp."
)
backend = get_attn_backend()
from sglang.srt.layers.attention.dots_hybrid_backend import (
DotsHybridAttnBackend,
DotsSWAMLAAttnBackend,
)
from sglang.srt.layers.attention.hybrid_attn_backend import HybridAttnBackend
if isinstance(backend, DotsHybridAttnBackend) and self.use_swa:
backend = backend.selected_swa_backend(forward_batch)
elif isinstance(backend, HybridAttnBackend):
backend = backend._select_backend(forward_batch.forward_mode)
if isinstance(backend, DotsSWAMLAAttnBackend):
backend = backend.selected_backend(forward_batch)
backend_name = type(backend).__name__.lower()
if "flashattention" in backend_name:
attention_backend = "fa3"
elif "flashmla" in backend_name:
attention_backend = "flashmla"
elif "triton" in backend_name:
attention_backend = "triton"
elif "sparse" in backend_name or self.use_nsa:
attention_backend = "nsa"
else:
raise NotImplementedError(
f"Unsupported Dots3 attention backend: {type(backend).__name__}"
)
self.current_attention_backend = attention_backend
if attention_backend == "fa3":
if self.use_swa:
# Expanded SWA-MHA is prefill-only; other modes use paged MLA.
return (
Dots3AttnForwardMethod.MLA
if forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
or forward_batch.forward_mode.is_draft_extend_v2()
else Dots3AttnForwardMethod.SWA_MHA
)
if forward_batch.forward_mode.is_extend_without_speculative():
return Dots3AttnForwardMethod.MHA_CHUNKED_KV
else:
return Dots3AttnForwardMethod.MLA
elif attention_backend in ("flashmla", "triton", "nsa"):
return Dots3AttnForwardMethod.MLA
else:
raise NotImplementedError(
f"Dots3 only supports CUDA attention backends (fa3, triton, flashmla, nsa), got: {attention_backend}"
)
def _apply_attention_gate(self, attn_output: torch.Tensor, g: torch.Tensor):
if attn_output.ndim != 3:
attn_output = attn_output.reshape(-1, self.num_local_heads, self.v_head_dim)
g = torch.nn.functional.sigmoid(g)
return attn_output * (
g.unsqueeze(-1)
if self.attention_gate_type == "headwise"
else g.reshape(-1, self.num_local_heads, self.v_head_dim)
)
# TODO(xiaozhi): Fuse this to post_load_weights.
def _maybe_apply_lora_rescale(self, lora_tensor: torch.Tensor, lora_dim):
if not self.apply_mla_qkv_lora_rescale:
return lora_tensor
return lora_tensor * math.sqrt(self.hidden_size / lora_dim)
@staticmethod
def _absorbed_bmm(
lhs: torch.Tensor,
weight: torch.Tensor,
out: torch.Tensor,
weight_scale: Optional[torch.Tensor] = None,
) -> torch.Tensor:
if weight.dtype == torch.float8_e4m3fn:
import deep_gemm
assert weight_scale is not None
lhs_fp8 = per_token_group_quant_einsum_fp8(lhs)
deep_gemm.fp8_einsum(
"bhr,hdr->bhd",
lhs_fp8,
(weight, weight_scale),
out,
)
else:
torch.bmm(
lhs.transpose(0, 1),
weight,
out=out.transpose(0, 1),
)
return out
def _get_kv_write_loc(self, forward_batch):
swa_loc = None
if self.use_swa:
backend = get_attn_backend()
from sglang.srt.layers.attention.dots_hybrid_backend import (
DotsHybridAttnBackend,
DotsSWAMLAAttnBackend,
)
if isinstance(backend, (DotsHybridAttnBackend, DotsSWAMLAAttnBackend)):
backend.maybe_rebuild_metadata_after_dp_padding(forward_batch)
if isinstance(backend, DotsHybridAttnBackend):
backend = backend.selected_swa_backend(forward_batch)
metadata = backend.forward_metadata
swa_loc = metadata.swa_out_cache_loc
if (
swa_loc is None
or swa_loc.shape[0] != forward_batch.out_cache_loc.shape[0]
):
if isinstance(backend, DotsSWAMLAAttnBackend):
out_cache_loc = backend.select_draft_step_out_cache_loc(
forward_batch
)
else:
out_cache_loc = forward_batch.out_cache_loc
swa_loc = get_token_to_kv_pool().translate_loc_from_full_to_swa(
out_cache_loc
)
return KVWriteLoc.for_batch(forward_batch, swa_loc=swa_loc)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
zero_allocator: BumpAllocator,
):
s = self.forward_prepare(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
zero_allocator=zero_allocator,
)
return self.forward_core(s)
def forward_prepare(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
zero_allocator: BumpAllocator,
):
if hidden_states.shape[0] == 0:
assert not self.o_proj.reduce_results, (
"short-circuiting allreduce will lead to hangs"
)
return hidden_states, None, forward_batch, None
attn_forward_method = self.dispatch_attn_forward_method(forward_batch)
if attn_forward_method == Dots3AttnForwardMethod.MHA_CHUNKED_KV:
inner_state = self.forward_normal_chunked_kv_prepare(
positions, hidden_states, forward_batch, zero_allocator
)
elif attn_forward_method == Dots3AttnForwardMethod.SWA_MHA:
inner_state = self.forward_swa_mha_prepare(
positions, hidden_states, forward_batch, zero_allocator
)
elif attn_forward_method == Dots3AttnForwardMethod.MLA:
inner_state = self.forward_absorb_prepare(
positions, hidden_states, forward_batch, zero_allocator
)
else:
raise NotImplementedError
return None, attn_forward_method, forward_batch, inner_state
def forward_core(self, intermediate_state):
hidden_states, attn_forward_method, _, inner_state = intermediate_state
if inner_state is None:
return hidden_states
if attn_forward_method == Dots3AttnForwardMethod.MHA_CHUNKED_KV:
return self.forward_normal_chunked_kv_core(*inner_state)
elif attn_forward_method == Dots3AttnForwardMethod.SWA_MHA:
return self.forward_swa_mha_core(*inner_state)
elif attn_forward_method == Dots3AttnForwardMethod.MLA:
return self.forward_absorb_core(*inner_state)
else:
raise NotImplementedError
def forward_normal_prepare(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
zero_allocator: BumpAllocator,
):
q, latent_cache, g = self._split_fused_qkv_a_g_proj_out(
self.fused_qkv_a_g_proj_with_mqa(hidden_states)[0]
)
q = self.q_a_layernorm(q)
q = self._maybe_apply_lora_rescale(q, self.q_lora_rank)
q = self.q_b_proj(q)[0]
q = q.view(-1, self.num_local_heads, self.qk_head_dim)
_, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
kv_a, _ = latent_cache.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
latent_cache = latent_cache.unsqueeze(1)
kv_a = self.kv_a_layernorm(kv_a)
kv_a = self._maybe_apply_lora_rescale(kv_a, self.kv_lora_rank)
kv = self.kv_b_proj(kv_a)[0]
kv = kv.view(-1, self.num_local_heads, self.qk_nope_head_dim + self.v_head_dim)
k_nope = kv[..., : self.qk_nope_head_dim]
v = kv[..., self.qk_nope_head_dim :]
k_pe = latent_cache[:, :, self.kv_lora_rank :]
k_pe = self.k_rope_only_layernorm(k_pe)
q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
q[..., self.qk_nope_head_dim :] = q_pe
k = torch.empty_like(q)
k[..., : self.qk_nope_head_dim] = k_nope
k[..., self.qk_nope_head_dim :] = k_pe
latent_cache[:, :, : self.kv_lora_rank] = kv_a.unsqueeze(1)
latent_cache[:, :, self.kv_lora_rank :] = k_pe
# Save latent cache
get_token_to_kv_pool().set_kv_buffer(
self.attn_mha,
self._get_kv_write_loc(forward_batch),
latent_cache,
None,
)
return q, k, v, g, forward_batch
def forward_absorb_prepare(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
zero_allocator: BumpAllocator,
):
from sglang.srt.model_executor.runner_utils.capture_mode import (
get_is_capture_mode,
)
q, latent_cache, g = self._split_fused_qkv_a_g_proj_out(
self.fused_qkv_a_g_proj_with_mqa(hidden_states)[0]
)
k_nope = latent_cache[..., : self.kv_lora_rank]
# overlap qk norm
if self.alt_stream is not None and get_is_capture_mode():
current_stream = torch.cuda.current_stream()
self.alt_stream.wait_stream(current_stream)
q = self.q_a_layernorm(q)
q = self._maybe_apply_lora_rescale(q, self.q_lora_rank)
with torch.cuda.stream(self.alt_stream):
k_nope = self.kv_a_layernorm(k_nope)
k_nope = self._maybe_apply_lora_rescale(k_nope, self.kv_lora_rank)
current_stream.wait_stream(self.alt_stream)
else:
q = self.q_a_layernorm(q)
q = self._maybe_apply_lora_rescale(q, self.q_lora_rank)
k_nope = self.kv_a_layernorm(k_nope)
k_nope = self._maybe_apply_lora_rescale(k_nope, self.kv_lora_rank)
# NSA indexer owns its projections and uses the standard SGLang layer.
topk_indices = None
if self.use_nsa:
topk_indices = self.indexer(
x=hidden_states,
q_lora=q,
positions=positions,
forward_batch=forward_batch,
layer_id=self.layer_id,
)
k_nope = k_nope.unsqueeze(1)
q = self.q_b_proj(q)[0]
q = q.view(-1, self.num_local_heads, self.qk_head_dim)
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
k_pe = latent_cache[..., self.kv_lora_rank :].unsqueeze(1)
k_pe = self.k_rope_only_layernorm(k_pe)
q_nope_out = q_nope.new_empty(
(q_nope.shape[0], self.num_local_heads, self.kv_lora_rank)
)
self._absorbed_bmm(q_nope, self.w_kc, q_nope_out, self.w_scale_k)
q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
return (
q_pe,
k_pe,
q_nope_out,
k_nope,
g,
forward_batch,
zero_allocator,
topk_indices,
)
def forward_absorb_core(
self,
q_pe,
k_pe,
q_nope_out,
k_nope,
g,
forward_batch,
zero_allocator,
topk_indices=None,
):
attn_kwargs = {}
if topk_indices is not None:
attn_kwargs["topk_indices"] = topk_indices
from sglang.srt.layers.attention.dots_hybrid_backend import (
DotsHybridAttnBackend,
DotsSWAMLAAttnBackend,
)
backend = get_attn_backend()
if (
self.use_swa
and (
isinstance(backend, DotsHybridAttnBackend)
or (
isinstance(backend, DotsSWAMLAAttnBackend)
and backend.uses_flash_attention(forward_batch)
)
)
and (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
or forward_batch.forward_mode.is_draft_extend_v2()
)
):
get_token_to_kv_pool().set_mla_kv_buffer(
self.attn_mqa,
self._get_kv_write_loc(forward_batch),
k_nope,
k_pe,
)
q_input = torch.cat([q_nope_out, q_pe], dim=-1)
attn_output = backend.forward_swa_mla_absorbed(
q_input, self.attn_mqa, forward_batch
)
elif self.current_attention_backend in ("fa3", "nsa") or self.use_nsa:
assert (
not self.use_nsa
or topk_indices is not None
or forward_batch.forward_mode.is_idle()
), "NSA attention requires topk_indices from indexer."
get_token_to_kv_pool().set_mla_kv_buffer(
self.attn_mqa,
self._get_kv_write_loc(forward_batch),
k_nope,
k_pe,
)
attn_output = self.attn_mqa(
q_nope_out,
k_nope,
k_nope,
forward_batch,
q_rope=q_pe,
k_rope=k_pe,
save_kv_cache=False,
**attn_kwargs,
)
elif self.current_attention_backend in {"triton", "flashmla"}:
q_input = torch.cat([q_nope_out, q_pe], dim=-1)
k_input = torch.cat([k_nope, k_pe], dim=-1)
v_input = k_nope.contiguous()
get_token_to_kv_pool().set_mla_kv_buffer(
self.attn_mqa,
self._get_kv_write_loc(forward_batch),
k_nope,
k_pe,
)
attn_output = self.attn_mqa(
q_input, k_input, v_input, forward_batch, save_kv_cache=False
)
attn_output = attn_output.view(-1, self.num_local_heads, self.kv_lora_rank)
attn_bmm_output = attn_output.new_empty(
(attn_output.shape[0], self.num_local_heads, self.v_head_dim),
)
self._absorbed_bmm(attn_output, self.w_vc, attn_bmm_output, self.w_scale_v)
attn_bmm_output = self._apply_attention_gate(attn_bmm_output, g)
output, _ = self.o_proj(
attn_bmm_output.reshape(-1, self.num_local_heads * self.v_head_dim)
)
return output
def _chunked_prefix_attn_mha(
self,
q: torch.Tensor,
accum_output: torch.Tensor,
accum_lse: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
assert forward_batch.num_prefix_chunks is not None
for i in range(forward_batch.num_prefix_chunks):
forward_batch.set_prefix_chunk_idx(i)
# Fetch latent cache from memory pool with precomputed chunked kv indices
latent_cache_buf = get_token_to_kv_pool().get_key_buffer(
self.attn_mha.layer_id
)
latent_cache = latent_cache_buf[
forward_batch.prefix_chunk_kv_indices[i]
].contiguous()
kv_a_normed, k_pe = latent_cache.split(
[self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
)
kv_a_normed = kv_a_normed.squeeze(1).contiguous()
kv = self.kv_b_proj(kv_a_normed)[0]
kv = kv.view(
-1, self.num_local_heads, self.qk_nope_head_dim + self.v_head_dim
)
v = kv[..., self.qk_nope_head_dim :]
k_nope = kv[..., : self.qk_nope_head_dim]
k = torch.empty(
(
k_nope.shape[0],
self.num_local_heads,
self.qk_nope_head_dim + self.qk_rope_head_dim,
),
dtype=v.dtype,
device=v.device,
)
k[..., : self.qk_nope_head_dim] = k_nope
k[..., self.qk_nope_head_dim :] = k_pe
output, lse = self.attn_mha(q, k, v, forward_batch, save_kv_cache=False)
tmp_output = torch.empty_like(accum_output)
tmp_lse = torch.empty_like(accum_lse)
merge_state_v2(output, lse, accum_output, accum_lse, tmp_output, tmp_lse)
accum_output, accum_lse = tmp_output, tmp_lse
return accum_output
def forward_normal_chunked_kv_prepare(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
zero_allocator: BumpAllocator,
):
# Materialize long prefixes in chunks to limit expanded K/V memory.
return self.forward_normal_prepare(
positions, hidden_states, forward_batch, zero_allocator
)
def forward_normal_chunked_kv_core(self, q, k, v, g, forward_batch):
has_extend_prefix = any(forward_batch.extend_prefix_lens_cpu)
# Only initialize the info once
if has_extend_prefix and forward_batch.num_prefix_chunks is None:
forward_batch.prepare_chunked_prefix_cache_info(q.device)
backend = get_attn_backend()
from sglang.srt.layers.attention.dots_hybrid_backend import (
DotsHybridAttnBackend,
DotsSWAMLAAttnBackend,
)
if isinstance(backend, (DotsHybridAttnBackend, DotsSWAMLAAttnBackend)):
backend.init_mha_chunk_metadata(forward_batch)
# Zero padded V rows because FA3 tiles may read past cu_seqlens_q[-1].
real_num_tokens = forward_batch.extend_num_tokens
if real_num_tokens is not None and v.shape[0] > real_num_tokens:
v[real_num_tokens:] = 0
forward_batch.mha_return_lse = has_extend_prefix
# Do mha for extended part without prefix
forward_batch.set_attn_attend_prefix_cache(False)
attn_output = self.attn_mha(q, k, v, forward_batch, save_kv_cache=False)
# Do mha attention with chunked prefix cache if there are any sequence with prefix
if has_extend_prefix:
attn_output, lse = attn_output
forward_batch.set_attn_attend_prefix_cache(True)
attn_output = self._chunked_prefix_attn_mha(
q=q,
accum_output=attn_output,
accum_lse=lse,
forward_batch=forward_batch,
)
attn_output = self._apply_attention_gate(attn_output, g)
attn_output = attn_output.reshape(-1, self.num_local_heads * self.v_head_dim)
output, _ = self.o_proj(attn_output)
return output
def forward_swa_mha_prepare(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
zero_allocator: BumpAllocator,
):
q_lora, latent_cache, g = self._split_fused_qkv_a_g_proj_out(
self.fused_qkv_a_g_proj_with_mqa(hidden_states)[0]
)
q_lora = self.q_a_layernorm(q_lora)
q_lora = self._maybe_apply_lora_rescale(q_lora, self.q_lora_rank)
q = self.q_b_proj(q_lora)[0]
q = q.view(-1, self.num_local_heads, self.qk_head_dim)
kv_a = latent_cache[..., : self.kv_lora_rank]
kv_a = self.kv_a_layernorm(kv_a)
kv_a = self._maybe_apply_lora_rescale(kv_a, self.kv_lora_rank)
q_pe = q[..., -self.qk_rope_head_dim :]
latent_cache = latent_cache.unsqueeze(1)
k_pe = latent_cache[..., -self.qk_rope_head_dim :]
k_pe = self.k_rope_only_layernorm(k_pe)
q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
q[..., self.qk_nope_head_dim :] = q_pe
latent_cache[..., : self.kv_lora_rank] = kv_a.unsqueeze(1)
latent_cache[..., self.kv_lora_rank :] = k_pe
# Save latent cache
get_token_to_kv_pool().set_kv_buffer(
self.attn_mha,
self._get_kv_write_loc(forward_batch),
latent_cache,
None,
)
# Load full latent cache
backend = get_attn_backend()
from sglang.srt.layers.attention.dots_hybrid_backend import (
DotsHybridAttnBackend,
DotsSWAMLAAttnBackend,
)
assert isinstance(backend, (DotsHybridAttnBackend, DotsSWAMLAAttnBackend))
latent_cache = backend.get_swa_mla_prefill_latent_cache(
forward_batch, self.attn_mha.layer_id
)
kv_a, k_pe = latent_cache.split(
[self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
)
# `split` keeps the 576-wide latent-cache stride while this projection
# consumes only the 512-wide LoRA slice. Block-FP8 activation
# quantization requires a contiguous input.
kv = self.kv_b_proj(kv_a.contiguous())[0]
kv = kv.view(-1, self.num_local_heads, self.qk_nope_head_dim + self.v_head_dim)
k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
kv_len_sum = latent_cache.shape[0]
k = torch.empty(
(
kv_len_sum,
self.num_local_heads,
self.qk_nope_head_dim + self.qk_rope_head_dim,
),
dtype=q.dtype,
device=q.device,
)
k[..., : self.qk_nope_head_dim] = k_nope
k[..., self.qk_nope_head_dim :] = k_pe
return q, k, v, g, forward_batch
def forward_swa_mha_core(self, q, k, v, g, forward_batch):
backend = get_attn_backend()
from sglang.srt.layers.attention.dots_hybrid_backend import (
DotsHybridAttnBackend,
DotsSWAMLAAttnBackend,
)
assert isinstance(backend, (DotsHybridAttnBackend, DotsSWAMLAAttnBackend))
attn_output = backend.forward_swa_mla_expanded(
q, k, v, self.attn_mha, forward_batch
)
attn_output = self._apply_attention_gate(attn_output, g)
attn_output = attn_output.reshape(-1, self.num_local_heads * self.v_head_dim)
return self.o_proj(attn_output)[0]
class Dots3DecoderLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
layer_id: int,
quant_config: Optional[QuantizationConfig] = None,
is_nextn: bool = False,
prefix: str = "",
alt_stream: Optional[torch.cuda.Stream] = None,
) -> None:
super().__init__()
_require_cuda()
self.hidden_size = config.hidden_size
self.config = config
self.layer_id = layer_id
self.self_attn = Dots3AttentionMLA(
config=config,
quant_config=quant_config,
layer_id=layer_id,
reduce_results=False,
prefix=add_prefix("self_attn", prefix),
alt_stream=alt_stream,
)
attn, ffn = self.stage_facts(config, layer_id, is_nextn=is_nextn)
self.is_layer_sparse = ffn.sparse
if self.is_layer_sparse:
self.mlp = Dots3MoE(
config=config,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
layer_id=self.layer_id,
alt_stream=alt_stream,
is_nextn=is_nextn,
)
else:
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = Dots3MLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
parallel_group=mlp_parallel_group,
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_boundary, self.ffn_boundary = append_stages(
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
)
@classmethod
def stage_facts(
cls, config: PretrainedConfig, layer_id: int, is_nextn: bool = False
):
"""The stages the layer at ``layer_id`` declares, from the config: the
model's shared declaration function (see make_layers)."""
return (
declare_attn(),
declare_ffn(
sparse=cls._is_layer_sparse(config, layer_id, is_nextn=is_nextn),
next_layer_sparse=cls._is_layer_sparse(
config, layer_id + 1, is_nextn=False
),
),
)
@staticmethod
def _is_layer_sparse(
config: PretrainedConfig, layer_id: int, is_nextn: bool
) -> bool:
# The MTP block uses a dense MLP.
if is_nextn:
return False
return (
config.n_routed_experts is not None
and layer_id >= config.first_k_dense_replace
and layer_id % config.moe_layer_freq == 0
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
zero_allocator: BumpAllocator,
) -> torch.Tensor:
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
zero_allocator=zero_allocator,
)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states, forward_batch)
return self.ffn_boundary.finish(hidden_states, forward_batch)
class Dots3Model(nn.Module):
fall_back_to_pt_during_load = False
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
super().__init__()
_require_cuda()
self.first_k_dense_replace = config.first_k_dense_replace
self.pp_group = get_parallel().pp_group
if self.pp_group.is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
enable_tp=not is_dp_attention_enabled(),
)
else:
self.embed_tokens = PPMissingLayer()
self.alt_stream = torch.cuda.Stream() if _is_cuda else None
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: Dots3DecoderLayer(
config=config,
layer_id=idx,
quant_config=quant_config,
prefix=prefix,
alt_stream=self.alt_stream,
),
prefix=add_prefix("layers", prefix),
stage_facts=lambda idx: Dots3DecoderLayer.stage_facts(config, idx),
)
if self.pp_group.is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
self.norm = PPMissingLayer(return_tuple=True)
def get_input_embeddings(self) -> torch.Tensor:
return self.embed_tokens
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: torch.Tensor = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> Union[torch.Tensor, PPProxyTensors]:
total_num_layers = self.end_layer - self.start_layer
device = input_embeds.device if input_embeds is not None else input_ids.device
zero_allocator = BumpAllocator(
buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1),
dtype=torch.float32,
device=device,
)
if self.pp_group.is_first_rank:
if input_embeds is None:
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
normal_start_layer = self.start_layer
normal_end_layer = self.end_layer
if forward_batch.can_run_tbo:
if (
self.first_k_dense_replace > normal_start_layer
and self.first_k_dense_replace < normal_end_layer
):
normal_end_layer = self.first_k_dense_replace
elif self.first_k_dense_replace < normal_start_layer:
normal_end_layer = normal_start_layer = 0
for i in range(normal_start_layer, normal_end_layer):
with get_global_expert_distribution_recorder().with_current_layer(i):
layer = self.layers[i]
hidden_states = layer(
positions, hidden_states, forward_batch, zero_allocator
)
if normal_end_layer != self.end_layer:
hidden_states = model_forward_stages(
layers=self.layers[normal_end_layer : self.end_layer],
enable_tbo=True,
positions=positions,
forward_batch=forward_batch,
hidden_states=hidden_states,
zero_allocator=zero_allocator,
)
hidden_states = residual_batch.complete_output(hidden_states, forward_batch)
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
else:
if not forward_batch.forward_mode.is_idle():
hidden_states = residual_batch.final_norm(
hidden_states, forward_batch, self.norm
)
return hidden_states
class Dots3LanguageModelForCausalLM(nn.Module):
# for quark model load
packed_modules_mapping = {}
fused_shared_experts_architecture = "Dots3NoteForCausalLM"
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
super().__init__()
# for quark model load
# Always fuse q_a_proj/kv_a_proj_with_mqa/g_proj when loading Dots3.
self.fuse_qkv_a_g_proj = True
assert config.q_lora_rank is not None, (
"Dots3 requires q_lora_rank to enable fused_qkv_a_g_proj_with_mqa loading."
)
if self.fuse_qkv_a_g_proj:
self.packed_modules_mapping["fused_qkv_a_g_proj_with_mqa"] = [
"q_a_proj",
"kv_a_proj_with_mqa",
"g_proj",
]
self.pp_group = get_parallel().pp_group
self.config = config
self.quant_config = quant_config
# The weight loader reads this after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.model = Dots3Model(
config, quant_config, prefix=add_prefix("model", prefix)
)
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=add_prefix("lm_head", prefix),
use_attn_tp_group=get_parallel().enable_dp_lm_head,
)
self.logits_processor = LogitsProcessor(config)
self._routed_experts_weights_of_layer = LazyValue(
lambda: {
layer_id: layer.mlp.get_moe_weights()
for layer_id, layer in enumerate(self.model.layers)
if isinstance(layer.mlp, Dots3MoE)
}
)
@property
def routed_experts_weights_of_layer(self):
return self._routed_experts_weights_of_layer.value
@classmethod
def shared_experts_fusion_disable_reason(cls, hf_config, quant_config):
"""Return why shared-expert fusion is unavailable, or ``None``."""
if (
not _is_cuda
or torch.cuda.get_device_capability("cuda") < (8, 0)
or hf_config.architectures[0] != cls.fused_shared_experts_architecture
or hf_config.n_routed_experts != 256
or hf_config.n_shared_experts != 1
):
return "Shared-expert fusion is unsupported for this Dots3 configuration."
if get_parallel().moe_ep_size > 1:
return "Dots3 shared-expert fusion is unsupported with expert parallelism."
if quant_config is not None and quant_config.get_name() == "w4afp8":
return (
"Dots3 W4AFP8 shared and routed experts use incompatible quantization."
)
return None
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
"""Shared experts fused as routed slots when fusion is on."""
return hf_config.n_shared_experts or 0
def get_input_embeddings(self) -> nn.Embedding:
return self.model.embed_tokens
def pad_input_ids(
self,
input_ids: array,
mm_inputs: MultimodalInputs,
**kwargs,
) -> array:
token_pairs = []
if mm_inputs.im_start_id is not None and mm_inputs.im_end_id is not None:
token_pairs.append((mm_inputs.im_start_id, mm_inputs.im_end_id))
if mm_inputs.audio_start_id is not None and mm_inputs.audio_end_id is not None:
token_pairs.append((mm_inputs.audio_start_id, mm_inputs.audio_end_id))
return pad_mm_token_pairs(
input_ids, mm_inputs, data_token_pairs=token_pairs or None
)
def _get_precomputed_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
embed_param = next(self.model.embed_tokens.parameters())
features = []
for item in items:
feature = item.precomputed_embeddings
if feature is None:
feature = item.feature
if isinstance(feature, list):
features.extend(feature)
else:
features.append(feature)
return torch.cat(
[
feature.to(
device=embed_param.device,
dtype=embed_param.dtype,
non_blocking=True,
)
for feature in features
],
dim=0,
)
get_image_feature = _get_precomputed_feature
get_audio_feature = _get_precomputed_feature
@torch.no_grad()
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: torch.Tensor = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> torch.Tensor:
if self.pp_group.is_first_rank and input_embeds is None:
hidden_states = general_mm_embed_routine(
input_ids=input_ids,
positions=positions,
forward_batch=forward_batch,
language_model=self.model,
multimodal_model=self,
data_embedding_funcs={
Modality.IMAGE: self.get_image_feature,
Modality.AUDIO: self.get_audio_feature,
},
pp_proxy_tensors=pp_proxy_tensors,
)
else:
hidden_states = self.model(
input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors
)
if self.pp_group.is_last_rank:
return self.logits_processor(
input_ids, hidden_states, self.lm_head, forward_batch
)
else:
return hidden_states
@property
def start_layer(self):
return self.model.start_layer
@property
def end_layer(self):
return self.model.end_layer
def post_load_weights(self, is_nextn=False, weight_names=None):
# Perform post-processing after loading weights
if is_nextn:
# K MTP heads — one self_attn per head module.
num_nextn_layers = self.config.num_nextn_predict_layers
head_attns = [
self.model.heads[k].decoder.self_attn for k in range(num_nextn_layers)
]
else:
if weight_names is None:
layer_ids = range(self.model.start_layer, self.model.end_layer)
else:
layer_ids = set()
for name in weight_names:
if "kv_b_proj" in name:
layer_id = int(name.split(".")[2])
if layer_id < self.config.num_hidden_layers:
layer_ids.add(layer_id)
head_attns = None # not used for non-nextn path
# Iterate either over base-model layer ids OR over the K MTP head self_attns.
iterator = head_attns if is_nextn else layer_ids
for item in iterator:
self_attn = item if is_nextn else self.model.layers[item].self_attn
w = self_attn.kv_b_proj.weight
# Only two kv_b_proj formats are supported:
# 1) BF16 weights.
# 2) FP8 block-wise weights with block_size=(128, 128), consumed by DeepGEMM grouped kernels.
weight_block_size = None
block_scale = None
if w.dtype == torch.float8_e4m3fn:
assert (
self.quant_config is not None
and self.quant_config.weight_block_size is not None
), (
"Dots3 MLA kv_b_proj only supports FP8 block quantization with weight_block_size=(128, 128)."
)
weight_block_size = tuple(self.quant_config.weight_block_size)
assert weight_block_size == (
128,
128,
), (
f"Dots3 MLA kv_b_proj only supports FP8 block_size=(128, 128), got {weight_block_size}."
)
block_scale = self_attn.kv_b_proj.weight_scale_inv
if not (
self_attn.qk_nope_head_dim % weight_block_size[0] == 0
and self_attn.v_head_dim % weight_block_size[0] == 0
and envs.SGLANG_USE_DEEPGEMM_BMM.get()
):
# NOTE(xiaozhi): At this point, we do not requant but change to use BF16.
# The issue is that when multiplying w_kc the reduction dim can not be divided
# by 128. May modify the DeepGEMM kernel to support this.
w = block_quant_dequant(
w,
block_scale,
weight_block_size,
torch.bfloat16,
)
else:
assert w.dtype == torch.bfloat16, (
f"Dots3 MLA kv_b_proj only supports BF16 or FP8(128x128), got dtype={w.dtype}."
)
w_kc, w_vc = w.unflatten(
0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim)
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
if w.dtype == torch.bfloat16:
self_attn.w_kc = bind_or_assign(self_attn.w_kc, w_kc)
self_attn.w_vc = bind_or_assign(self_attn.w_vc, w_vc.transpose(1, 2))
else:
num_tiles_k = self_attn.qk_nope_head_dim // weight_block_size[0]
num_tiles_n = self_attn.v_head_dim // weight_block_size[0]
ws_kc, ws_vc = block_scale.unflatten(
0, (-1, (num_tiles_k + num_tiles_n))
).split([num_tiles_k, num_tiles_n], dim=1)
self_attn.w_scale_k = bind_or_assign(
self_attn.w_scale_k, ws_kc.transpose(1, 2).contiguous()
)
self_attn.w_scale_v = bind_or_assign(
self_attn.w_scale_v, ws_vc.contiguous()
)
self_attn.w_kc = bind_or_assign(
self_attn.w_kc, w_kc.transpose(1, 2).contiguous()
)
self_attn.w_vc = bind_or_assign(self_attn.w_vc, w_vc)
if (
deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM
and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
and self.quant_config is not None
and self.quant_config.weight_block_size is not None
):
self._weight_requant_ue8m0(is_nextn)
def _weight_requant_ue8m0(self, is_nextn=False):
weight_block_size = self.quant_config.weight_block_size
moe_layers = list(
range(
self.config.first_k_dense_replace,
self.config.num_hidden_layers,
self.config.moe_layer_freq,
)
)
num_nextn_layers = self.config.num_nextn_predict_layers
num_hidden_layers = (
num_nextn_layers if is_nextn else self.config.num_hidden_layers
)
for layer_id in range(num_hidden_layers):
if is_nextn:
layer = self.model.heads[layer_id].decoder
else:
layer = self.model.layers[layer_id]
for module in [
layer.self_attn.fused_qkv_a_g_proj_with_mqa,
layer.self_attn.q_b_proj,
layer.self_attn.kv_b_proj,
layer.self_attn.o_proj,
]:
requant_weight_ue8m0_inplace(
module.weight, module.weight_scale_inv, weight_block_size
)
if layer_id in moe_layers or is_nextn:
if (
isinstance(layer.mlp, Dots3MoE)
and layer.mlp.shared_experts is not None
):
shared_experts = layer.mlp.shared_experts
for module in [
shared_experts.gate_up_proj,
shared_experts.down_proj,
]:
requant_weight_ue8m0_inplace(
module.weight, module.weight_scale_inv, weight_block_size
)
experts = layer.mlp.experts
if isinstance(experts, DeepEPMoE):
for w in [
experts.w13_weight_fp8,
experts.w2_weight_fp8,
]:
requant_weight_ue8m0_inplace(w[0], w[1], weight_block_size)
else:
mlp = layer.mlp
assert isinstance(mlp, Dots3MLP)
for module in [
mlp.gate_up_proj,
mlp.down_proj,
]:
requant_weight_ue8m0_inplace(
module.weight, module.weight_scale_inv, weight_block_size
)
def load_weights(
self,
weights: Iterable[Tuple[str, torch.Tensor]],
is_nextn=False,
extra_params_mapping=None,
):
if is_nextn:
num_nextn_layers = self.config.num_nextn_predict_layers
# compatible with old design: when the main model has only 1 layer,
# the MTP layer is at id 0; otherwise it starts at num_hidden_layers
# and spans num_nextn_layers consecutive layer ids.
base_nextn_layer_id = (
0
if self.config.num_hidden_layers == 1
else self.config.num_hidden_layers
)
# Set of source layer prefixes for K MTP heads.
nextn_layer_ids = list(
range(base_nextn_layer_id, base_nextn_layer_id + num_nextn_layers)
)
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
if extra_params_mapping is not None:
stacked_params_mapping.extend(extra_params_mapping)
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = FusedMoE.make_expert_params_mapping(
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts + self.num_fused_shared_experts,
)
# Map per-expert input scales used by mixed-precision checkpoints.
if self.quant_config and self.quant_config.get_name() == "w4afp8":
expert_params_mapping += FusedMoE.make_expert_input_scale_params_mapping(
num_experts=self.config.n_routed_experts
)
# Always fuse q_a_proj/kv_a_proj_with_mqa/g_proj when loading Dots3.
fuse_qkv_a_g_proj = True
assert self.config.q_lora_rank is not None, (
"Dots3 requires q_lora_rank to enable fused_qkv_a_g_proj_with_mqa loading."
)
cached_a_proj = {} if fuse_qkv_a_g_proj else None
def shard_g_proj_for_attention_tp(
weight: torch.Tensor,
cat_dim: int,
is_scale: bool,
*,
q_b_proj: ColumnParallelLinear,
):
group = unwrap_lora_layer(q_b_proj).tp_group
attn_tp_rank = group.rank_in_group if group is not None else 0
attn_tp_size = group.world_size if group is not None else 1
assert weight.ndim > cat_dim, (
f"weight.ndim={weight.ndim}, cat_dim={cat_dim}"
)
dim_size = weight.shape[cat_dim]
if not is_scale:
assert dim_size % attn_tp_size == 0, (
f"dim_size={dim_size} is not divisible by attn_tp_size="
f"{attn_tp_size}"
)
start_idx = (attn_tp_rank * dim_size) // attn_tp_size
end_idx = ceil_div((attn_tp_rank + 1) * dim_size, attn_tp_size)
return weight.narrow(cat_dim, start_idx, end_idx - start_idx)
def align_tensor(
weight: torch.Tensor,
cat_dim: int,
align_size: int,
) -> torch.Tensor:
original_size = weight.shape[cat_dim]
aligned_size = ceil_align(original_size, align_size)
if aligned_size == original_size:
return weight
pad_shape = list(weight.shape)
pad_shape[cat_dim] = aligned_size - original_size
return torch.cat([weight, weight.new_zeros(pad_shape)], dim=cat_dim)
def _try_load_fused_qkv(
q_a_proj_name: str,
kv_a_proj_name: str,
g_proj_name: str,
param_name: str,
cat_dim: int,
) -> bool:
if not (
q_a_proj_name in cached_a_proj
and kv_a_proj_name in cached_a_proj
and g_proj_name in cached_a_proj
):
return False
if param_name not in params_dict:
raise ValueError(f"{param_name} not found in params_dict.")
param = params_dict[param_name]
target_dim = param.shape[cat_dim] if param.dim() > cat_dim else None
is_scale = param_name.endswith(".weight_scale_inv")
q_a_proj_weight = cached_a_proj[q_a_proj_name]
kv_a_proj_weight = cached_a_proj[kv_a_proj_name]
g_proj_weight = cached_a_proj[g_proj_name]
g_proj_shard = shard_g_proj_for_attention_tp(
g_proj_weight,
cat_dim,
is_scale,
q_b_proj=self.get_submodule(param_name.rsplit(".", 2)[0]).q_b_proj,
)
scale_block_n = _get_scale_block_n(self.quant_config)
kv_a_proj_weight_aligned = (
align_tensor(kv_a_proj_weight, cat_dim, scale_block_n)
if not is_scale
else kv_a_proj_weight
)
g_proj_shard_aligned = (
align_tensor(g_proj_shard, cat_dim, max(8, scale_block_n))
if not is_scale
else g_proj_shard
)
fused_parts = [
q_a_proj_weight,
kv_a_proj_weight_aligned,
g_proj_shard_aligned,
]
fused_weight = torch.cat(fused_parts, dim=cat_dim)
fused_dim = fused_weight.shape[cat_dim]
if target_dim is not None and fused_dim != target_dim:
raise ValueError(
f"Cannot match fused_qkv_a_g_proj_with_mqa shape for {param_name}: "
f"target_dim={target_dim}, fused_dim={fused_dim}."
)
weight_loader = _get_param_weight_loader(param)
futures.append(executor.submit(weight_loader, param, fused_weight))
cached_a_proj.pop(q_a_proj_name)
cached_a_proj.pop(kv_a_proj_name)
cached_a_proj.pop(g_proj_name)
return True
if is_nextn:
nextn_layer_prefixes = [f"model.layers.{lid}" for lid in nextn_layer_ids]
nextn_spec_weight_names = [
"shared_head.norm",
"eh_proj",
"enorm",
"hnorm",
]
# Map source layer id -> head index (0..num_nextn_layers-1).
def _match_nextn_prefix(name: str):
for k, p in enumerate(nextn_layer_prefixes):
# require dot-boundary: layer 30 vs 300
if name == p or name.startswith(p + "."):
return k, p
return None, None
if self.num_fused_shared_experts > 0:
assert self.num_fused_shared_experts == 1
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = []
params_dict = dict(self.named_parameters())
pending_indexer_wk = {}
weight_names = []
for name, loaded_weight in weights:
layer_id = get_layer_id(name)
if (
layer_id is not None
and not is_nextn
and (
layer_id < self.model.start_layer
or layer_id >= self.model.end_layer
)
):
continue
if self.num_fused_shared_experts > 0 and "mlp.shared_experts" in name:
name = name.replace(
"mlp.shared_experts",
f"mlp.experts.{self.config.n_routed_experts}",
)
weight_names.append(name)
if not is_nextn:
num_nextn_layers = self.config.num_nextn_predict_layers
if num_nextn_layers > 0 and name.startswith("model.layers"):
name_list = name.split(".")
if (
len(name_list) >= 3
and int(name_list[2]) >= self.config.num_hidden_layers
):
continue
# The NextN draft model owns model.mtp.* weights.
if num_nextn_layers > 0 and name.startswith("model.mtp."):
continue
else:
# Remap the MTP-specific embedding into the draft model.
if name == "model.mtp.embed_tokens.weight":
name = "model.embed_tokens.weight"
elif name == "model.embed_tokens.weight":
# The target model owns the main embedding.
continue
else:
head_idx, matched_prefix = _match_nextn_prefix(name)
if matched_prefix is None:
continue
# Remap an MTP head into the draft model's shared head.
if "shared_head.head" in name:
name = name.replace(
matched_prefix, "model.shared_head.head"
)
param = params_dict.get(name)
if param is None:
continue
weight_loader = _get_param_weight_loader(param)
futures.append(
executor.submit(weight_loader, param, loaded_weight)
)
continue
is_decoder = True
# nextn-specific adapters (enorm/hnorm/eh_proj/shared_head.norm)
for weight_name in nextn_spec_weight_names:
if weight_name in name:
name = name.replace(
matched_prefix, f"model.heads.{head_idx}"
)
is_decoder = False
break
# MTP transformer block weights — go under
# model.heads.{k}.decoder.* (with shared block for "block"/"full"
# sharing the duplicate writes converge on the same parameter).
if is_decoder:
name = name.replace(
matched_prefix, f"model.heads.{head_idx}.decoder"
)
if "rotary_emb.inv_freq" in name:
continue
if (
".indexer.wk." in name or ".indexer.weights_proj." in name
) and _load_fused_indexer_wk(
name,
loaded_weight,
params_dict,
pending_indexer_wk,
self.quant_config,
):
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if weight_name not in name:
continue
# We have mlp.experts[0].gate_proj in the checkpoint.
# Since we handle the experts below in expert_params_mapping,
# we need to skip here BEFORE we update the name, otherwise
# name will be updated to mlp.experts[0].gate_up_proj, which
# will then be updated below in expert_params_mapping
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
if ("mlp.experts." in name) and name not in params_dict:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
futures.append(
executor.submit(weight_loader, param, loaded_weight, shard_id)
)
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
futures.append(
executor.submit(
weight_loader,
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip loading embed_tokens if not first rank in pipeline parallelism
if ".embed_tokens." in name and not self.pp_group.is_first_rank:
continue
# Skip loading norm if not last rank in pipeline parallelism
if ".norm." in name and not self.pp_group.is_last_rank:
continue
if (
fuse_qkv_a_g_proj
and ".self_attn." in name
and (
"q_a_proj" in name
or "kv_a_proj_with_mqa" in name
or "g_proj" in name
)
):
cached_a_proj[name] = loaded_weight
if "q_a_proj" in name:
q_a_proj_name = name
kv_a_proj_name = name.replace(
"q_a_proj", "kv_a_proj_with_mqa"
)
g_proj_name = name.replace("q_a_proj", "g_proj")
param_name = name.replace(
"q_a_proj", "fused_qkv_a_g_proj_with_mqa"
)
elif "kv_a_proj_with_mqa" in name:
q_a_proj_name = name.replace(
"kv_a_proj_with_mqa", "q_a_proj"
)
kv_a_proj_name = name
g_proj_name = name.replace(
"kv_a_proj_with_mqa", "g_proj"
)
param_name = name.replace(
"kv_a_proj_with_mqa", "fused_qkv_a_g_proj_with_mqa"
)
else:
q_a_proj_name = name.replace("g_proj", "q_a_proj")
kv_a_proj_name = name.replace(
"g_proj", "kv_a_proj_with_mqa"
)
g_proj_name = name
param_name = name.replace(
"g_proj", "fused_qkv_a_g_proj_with_mqa"
)
cat_dim = 0
if self.quant_config is not None and (
self.quant_config.get_name() == "awq"
or self.quant_config.get_name() == "awq_marlin"
or self.quant_config.get_name() == "moe_wna16"
):
cat_dim = 1
_try_load_fused_qkv(
q_a_proj_name,
kv_a_proj_name,
g_proj_name,
param_name,
cat_dim,
)
else:
if (
"k_scale" in name or "v_scale" in name
) and name not in params_dict:
# modelopt attn kv scale is named differently
for scale in ["k_scale", "v_scale"]:
if scale in name:
name = name.replace(
f"{scale[0]}_proj", "attn_mqa"
)
break
if name not in params_dict:
# modelopt ckpt contains not needed weights for MTP module:
# model.decoder.self_attn.attn_mqa.v_scale and
# model.decoder.self_attn.attn_mqa.k_scale
logger.warning(f"{name} not found in params_dict.")
continue
param = params_dict[name]
weight_loader = _get_param_weight_loader(param)
futures.append(
executor.submit(weight_loader, param, loaded_weight)
)
if fuse_qkv_a_g_proj and cached_a_proj:
unresolved = sorted(cached_a_proj.keys())
preview = ", ".join(unresolved[:6])
extra = f" (+{len(unresolved) - 6} more)" if len(unresolved) > 6 else ""
raise ValueError(
"Unresolved fused q/kv/g projection weights while loading "
"fused_qkv_a_g_proj_with_mqa. Missing counterparts or unexpected "
f"names: {preview}{extra}"
)
if pending_indexer_wk:
unresolved = ", ".join(sorted(pending_indexer_wk.keys())[:6])
raise ValueError(
"Incomplete native DSA Indexer wk weights: " + unresolved
)
# Wait for all tasks to complete and raise any exceptions.
for future in concurrent.futures.as_completed(futures):
future.result()
self.post_load_weights(is_nextn=is_nextn, weight_names=weight_names)
def get_embed_and_head(self):
return self.model.embed_tokens.weight, self.lm_head.weight
def set_embed_and_head(self, embed, head):
# Share target embeddings and output head with the draft model.
del self.model.embed_tokens.weight
del self.lm_head.weight
self.model.embed_tokens.weight = embed
self.lm_head.weight = head
torch.cuda.empty_cache()
torch.cuda.synchronize()
def get_attention_sliding_window_size(self):
return get_attention_sliding_window_size(self.config)
@classmethod
def get_model_config_for_expert_location(cls, config):
return ModelConfigForExpertLocation(
num_layers=config.num_hidden_layers,
num_logical_experts=config.n_routed_experts,
num_groups=None,
)
class DotsNoteOmniThinkerForConditionalGeneration(nn.Module):
"""Dots thinker with in-process audio, vision, and language submodels."""
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
from pathlib import Path
from sglang.srt.models.dots3_common.dots_omni_towers import (
DotsNoteOmniAudioEncoder,
DotsNoteOmniVisionEncoder,
)
self.config = config
self.pp_group = get_parallel().pp_group
model_dir = Path(config._name_or_path)
self.language_model = Dots3LanguageModelForCausalLM(
config,
quant_config=quant_config,
prefix=add_prefix("language_model", prefix),
)
# Load multimodal towers only where embeddings are produced.
if self.pp_group.is_first_rank and not config.language_only:
self.audio_tower = DotsNoteOmniAudioEncoder(str(model_dir))
self.visual = DotsNoteOmniVisionEncoder(str(model_dir))
else:
self.audio_tower = None
self.visual = None
@property
def model(self):
return self.language_model.model
@property
def lm_head(self):
return self.language_model.lm_head
@property
def logits_processor(self):
return self.language_model.logits_processor
@property
def routed_experts_weights_of_layer(self):
return self.language_model.routed_experts_weights_of_layer
def get_input_embeddings(self):
return self.language_model.get_input_embeddings()
def pad_input_ids(self, input_ids: array, mm_inputs, **kwargs) -> array:
return self.language_model.pad_input_ids(input_ids, mm_inputs, **kwargs)
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
assert self.visual is not None
pixel_values = torch.cat([item.feature for item in items], dim=0).to(
device=self.visual.device,
dtype=self.visual.dtype,
non_blocking=True,
)
grid_thw = torch.cat([item.image_grid_thw for item in items], dim=0).to(
device=self.visual.device,
non_blocking=True,
)
return self.visual(pixel_values, grid_thw=grid_thw)
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
assert self.audio_tower is not None
waveforms = [
item.feature.to(
device=self.audio_tower.device,
dtype=torch.float32,
non_blocking=True,
)
for item in items
]
lengths = torch.tensor(
[waveform.numel() for waveform in waveforms],
dtype=torch.long,
)
features, token_lengths = self.audio_tower(waveforms, lengths)
expected = sum(
sum(end - start + 1 for start, end in item.offsets) for item in items
)
if features.shape[0] != expected or sum(token_lengths) != expected:
raise RuntimeError(
"Dots audio feature/token mismatch: "
f"features={features.shape[0]}, tower_lengths={token_lengths}, "
f"placeholders={expected}"
)
return features
@torch.no_grad()
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: torch.Tensor = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
):
if self.pp_group.is_first_rank and input_embeds is None:
hidden_states = general_mm_embed_routine(
input_ids=input_ids,
positions=positions,
forward_batch=forward_batch,
language_model=self.language_model.model,
multimodal_model=self,
data_embedding_funcs={
Modality.IMAGE: self.get_image_feature,
Modality.AUDIO: self.get_audio_feature,
},
pp_proxy_tensors=pp_proxy_tensors,
)
else:
hidden_states = self.language_model.model(
input_ids,
positions,
forward_batch,
input_embeds,
pp_proxy_tensors,
)
if self.pp_group.is_last_rank:
return self.language_model.logits_processor(
input_ids,
hidden_states,
self.language_model.lm_head,
forward_batch,
)
return hidden_states
class DotsNoteOmniForConditionalGeneration(nn.Module):
"""Native dots.note.omni conditional-generation model."""
packed_modules_mapping = Dots3LanguageModelForCausalLM.packed_modules_mapping
fall_back_to_pt_during_load = False
@staticmethod
def shared_experts_fusion_disable_reason(hf_config, quant_config):
return Dots3LanguageModelForCausalLM.shared_experts_fusion_disable_reason(
hf_config, quant_config
)
@staticmethod
def num_fusable_shared_experts(hf_config) -> int:
return Dots3LanguageModelForCausalLM.num_fusable_shared_experts(hf_config)
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
self.config = config
self.thinker = DotsNoteOmniThinkerForConditionalGeneration(
config,
quant_config=quant_config,
prefix=add_prefix("thinker", prefix),
)
language_model = self.thinker.language_model
self.pp_group = language_model.pp_group
self.quant_config = language_model.quant_config
self.num_fused_shared_experts = language_model.num_fused_shared_experts
self.forward = self.thinker.forward
self.pad_input_ids = self.thinker.pad_input_ids
@property
def model(self):
return self.thinker.language_model.model
@property
def lm_head(self):
return self.thinker.language_model.lm_head
@property
def logits_processor(self):
return self.thinker.language_model.logits_processor
@property
def routed_experts_weights_of_layer(self):
return self.thinker.language_model.routed_experts_weights_of_layer
@property
def start_layer(self):
return self.thinker.language_model.start_layer
@property
def end_layer(self):
return self.thinker.language_model.end_layer
def get_input_embeddings(self):
return self.thinker.get_input_embeddings()
def load_weights(self, weights, *args, **kwargs):
# Partition the flat checkpoint in one pass. This avoids reading the
# large tower tensors once through the model loader and again through
# safetensors.
load_towers = self.thinker.visual is not None
vision_state = {}
audio_state = {}
def language_weights():
for name, weight in weights:
if name.startswith("vision_encoder."):
if load_towers:
vision_state[name.removeprefix("vision_encoder.")] = weight
continue
if name.startswith("audio_encoder."):
if load_towers:
audio_state[name.removeprefix("audio_encoder.")] = weight
continue
yield name, weight
self.thinker.language_model.load_weights(language_weights(), *args, **kwargs)
if load_towers:
self.thinker.visual.load_converted_state(vision_state)
self.thinker.audio_tower.load_converted_state(audio_state)
def post_load_weights(self, *args, **kwargs):
return self.thinker.language_model.post_load_weights(*args, **kwargs)
def get_embed_and_head(self):
return self.thinker.language_model.get_embed_and_head()
def set_embed_and_head(self, embed, head):
return self.thinker.language_model.set_embed_and_head(embed, head)
def get_attention_sliding_window_size(self):
return self.thinker.language_model.get_attention_sliding_window_size()
@classmethod
def get_model_config_for_expert_location(cls, config):
return Dots3LanguageModelForCausalLM.get_model_config_for_expert_location(
config
)
class Dots3NoteForCausalLM(DotsNoteOmniForConditionalGeneration):
"""Canonical dots.note architecture exported by the flat checkpoint."""