[Refactor] Load auxiliary and expert checkpoints using owner partitions (#42589)

This commit is contained in:
Cheng Wan
2026-10-07 16:24:03 -07:00
committed by GitHub
parent 64c7f65eac
commit 5e12831cec
17 changed files with 1027 additions and 121 deletions
+2 -2
View File
@@ -431,6 +431,7 @@ class ScaledActivation(nn.Module):
self.act = act_module
self.input_is_parallel = input_is_parallel
if input_is_parallel:
self.tp_rank = get_parallel().tp_rank
tp_size = get_parallel().tp_size
intermediate_size_per_partition = divide(intermediate_size, tp_size)
else:
@@ -448,9 +449,8 @@ class ScaledActivation(nn.Module):
def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor):
param_data = param.data
if self.input_is_parallel:
tp_rank = get_parallel().tp_rank
shard_size = param_data.shape[0]
start_idx = tp_rank * shard_size
start_idx = self.tp_rank * shard_size
loaded_weight = loaded_weight.narrow(0, start_idx, shard_size)
assert param_data.shape == loaded_weight.shape
param_data.copy_(loaded_weight)
@@ -26,7 +26,6 @@ from sglang.srt.layers.quantization.utils import (
per_tensor_dequantize,
swap_w13_to_w31,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_bool_env_var, is_hip, set_weight_attrs
if TYPE_CHECKING:
@@ -98,7 +97,7 @@ class CompressedTensorsW8A8Fp8MoE(CompressedTensorsMoEScheme):
if self.block_quant:
assert self.weight_block_size is not None
layer.weight_block_size = self.weight_block_size
tp_size = get_parallel().tp_size
tp_size = layer.moe_tp_size
block_n, block_k = (
self.weight_block_size[0],
self.weight_block_size[1],
@@ -5,7 +5,6 @@ import re
import torch
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_cuda
_is_cuda = is_cuda()
@@ -61,33 +60,12 @@ def _load_gptoss_quark_expert_weights(model, weights, quark_expert_pat):
loaded_params: set[str] = set()
mxfp4_block = 32
moe_tp_rank = get_parallel().moe_tp_rank
moe_tp_size = get_parallel().moe_tp_size
moe_ep_rank = get_parallel().moe_ep_rank
moe_ep_size = get_parallel().moe_ep_size
intermediate_size = model.config.intermediate_size
assert intermediate_size % mxfp4_block == 0, (
f"{intermediate_size=} must be divisible by {mxfp4_block=}"
)
intermediate_size_block = intermediate_size // mxfp4_block
per_rank_intermediate_size_block = math.ceil(intermediate_size_block / moe_tp_size)
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
# Calculate common slicing bounds for current rank
assert model.config.num_local_experts % moe_ep_size == 0
moe_num_local_experts = model.config.num_local_experts // moe_ep_size
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
moe_tp_rank_end = min(
(moe_tp_rank + 1) * per_rank_intermediate_size, intermediate_size
)
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
for name, weight in weights:
# Quark stores experts separately as
# `experts.{N}.{gate_up_proj|down_proj}.{suffix}`; pull the
@@ -96,6 +74,30 @@ def _load_gptoss_quark_expert_weights(model, weights, quark_expert_pat):
if m is None:
continue
prefix, expert_str, proj, suffix = m.groups()
experts = model.get_submodule(prefix)
moe_tp_rank = experts.moe_tp_rank
moe_tp_size = experts.moe_tp_size
moe_ep_rank = experts.moe_ep_rank
moe_ep_size = experts.moe_ep_size
per_rank_intermediate_size_block = math.ceil(
intermediate_size_block / moe_tp_size
)
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
# Calculate common slicing bounds for current rank
assert model.config.num_local_experts % moe_ep_size == 0
moe_num_local_experts = model.config.num_local_experts // moe_ep_size
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
moe_tp_rank_end = min(
(moe_tp_rank + 1) * per_rank_intermediate_size, intermediate_size
)
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
global_expert_id = int(expert_str)
if global_expert_id < moe_ep_rank_start or global_expert_id >= moe_ep_rank_end:
continue
@@ -386,17 +386,17 @@ class BailingGroupRMSNormGate(RMSNormGated):
dtype=dtype,
activation="sigmoid",
)
self.tp_size = get_parallel().attn_tp_size
self.tp_rank = get_parallel().attn_tp_rank
self.weight.weight_loader = self.weight_loader
@staticmethod
def weight_loader(
self,
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
) -> None:
tp_size = get_parallel().attn_tp_size
tp_rank = get_parallel().attn_tp_rank
shard_size = loaded_weight.shape[0] // tp_size
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
shard_size = loaded_weight.shape[0] // self.tp_size
shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size)
param.data.copy_(loaded_weight[shard].contiguous())
return
+2 -2
View File
@@ -84,6 +84,7 @@ class LayerNorm(nn.Module):
def __init__(self, param_shape=None, eps=1e-5):
super().__init__()
self.weight = nn.Parameter(torch.ones(param_shape))
self.tp_rank = get_parallel().tp_rank if self.weight.dim() != 1 else 0
self.variance_epsilon = eps
set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
@@ -94,12 +95,11 @@ class LayerNorm(nn.Module):
return hidden_states, residuals
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
tp_rank = get_parallel().tp_rank
shard_dim = 0 if param.dim() != 1 else None
param_data = param.data
if shard_dim is not None:
shard_size = param_data.shape[shard_dim]
start_idx = tp_rank * shard_size
start_idx = self.tp_rank * shard_size
loaded_weight = loaded_weight.narrow(shard_dim, start_idx, shard_size)
assert param_data.shape == loaded_weight.shape
param_data.copy_(loaded_weight)
+2 -2
View File
@@ -103,6 +103,7 @@ class DbrxExperts(nn.Module):
):
super().__init__()
self.tp_size = get_parallel().tp_size
self.tp_rank = get_parallel().tp_rank
self.num_total_experts = config.ffn_config.moe_num_experts
self.top_k = config.ffn_config.moe_top_k
self.d_model = config.d_model
@@ -155,10 +156,9 @@ class DbrxExperts(nn.Module):
def weight_loader(
self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str
):
tp_rank = get_parallel().tp_rank
param_data = param.data
shard_size = self.intermediate_size
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size)
# DBRX uses GLU for each experts.
# GLU has 3 linear layers: w1, v1 and w2.
if weight_name.endswith("w1"):
+42 -26
View File
@@ -942,11 +942,6 @@ class GptOssForCausalLM(nn.Module):
loaded_params: set[str] = set()
mxfp4_block = 32
moe_tp_rank = get_parallel().moe_tp_rank
moe_tp_size = get_parallel().moe_tp_size
moe_ep_rank = get_parallel().moe_ep_rank
moe_ep_size = get_parallel().moe_ep_size
intermediate_size = self.config.intermediate_size
original_intermediate_size = getattr(
self.config, "original_intermediate_size", intermediate_size
@@ -956,30 +951,49 @@ class GptOssForCausalLM(nn.Module):
)
intermediate_size_block = intermediate_size // mxfp4_block
per_rank_intermediate_size_block = math.ceil(
intermediate_size_block / moe_tp_size
)
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
# Calculate common slicing bounds for current rank
assert self.config.num_local_experts % moe_ep_size == 0
moe_num_global_experts = self.config.num_local_experts
moe_num_local_experts = self.config.num_local_experts // moe_ep_size
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
moe_tp_rank_end = min(
(moe_tp_rank + 1) * per_rank_intermediate_size, original_intermediate_size
)
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
weight_device = next(iter(params_dict.values())).device
for name, weight in weights:
weight = weight.to(weight_device)
if not any(
suffix in name
for suffix in (
"gate_up_proj_blocks",
"down_proj_blocks",
"gate_up_proj_scales",
"down_proj_scales",
"gate_up_proj_bias",
"down_proj_bias",
)
):
continue
experts = self.get_submodule(name.rsplit(".", 1)[0])
moe_tp_rank = experts.moe_tp_rank
moe_tp_size = experts.moe_tp_size
moe_ep_rank = experts.moe_ep_rank
moe_ep_size = experts.moe_ep_size
per_rank_intermediate_size_block = math.ceil(
intermediate_size_block / moe_tp_size
)
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
# Calculate common slicing bounds for current rank
assert self.config.num_local_experts % moe_ep_size == 0
moe_num_global_experts = self.config.num_local_experts
moe_num_local_experts = self.config.num_local_experts // moe_ep_size
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
moe_tp_rank_end = min(
(moe_tp_rank + 1) * per_rank_intermediate_size,
original_intermediate_size,
)
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
if "gate_up_proj_blocks" in name:
# Handle MLP gate and up projection weights
new_name = name.replace("gate_up_proj_blocks", "w13_weight")
@@ -1240,8 +1254,10 @@ class GptOssForCausalLM(nn.Module):
weight_loader = param.weight_loader
if "bias" not in name:
loaded_weight = loaded_weight.transpose(-2, -1)
if "w2_weight_bias" in name and get_parallel().moe_tp_rank != 0:
loaded_weight = loaded_weight.zero_()
if "w2_weight_bias" in name:
experts = self.get_submodule(name.rsplit(".", 1)[0])
if experts.moe_tp_rank != 0:
loaded_weight = loaded_weight.zero_()
weight_loader(
param,
+39 -33
View File
@@ -100,29 +100,20 @@ STACKED_DENSE_PARAMS_MAPPING = [
("down_proj", "down_proj", None),
]
# Online RL weight-sync streams routed experts one at a time as FULL (unsharded)
# per-expert tensors named `...mlp.experts.{j}.gate_proj/up_proj/down_proj.weight`.
# Disk checkpoints only ever carry the fused w13_weight/w2_weight, so this pattern
# never fires on the ordinary loading path.
# Online RL updates use full per-expert gate/up/down tensors; disk checkpoints
# contain fused w13/w2 weights and do not match this pattern.
_PER_EXPERT_WEIGHT_RE = re.compile(
r"^(?P<pfx>.+\.mlp\.experts)\.(?P<eid>\d+)\.(?P<proj>gate_proj|up_proj|down_proj)\.weight$"
)
def _shard_full_to_local(
loaded_weight: torch.Tensor, dst: torch.Tensor, dim: int
loaded_weight: torch.Tensor, dst: torch.Tensor, dim: int, *, tp_rank: int
) -> torch.Tensor:
"""Slice a FULL (unsharded) per-expert weight to this MoE-TP rank's shard along `dim`.
The online weight-sync ships full per-expert tensors (parallelism-agnostic HF
layout); sglang owns its own MoE-TP sharding, so narrow here per
get_parallel().moe_tp_rank. With TP1 the dims already match and this is the
identity, so the ordinary path is byte-for-byte unchanged.
"""
"""Shard full HF per-expert updates with the owning module's MoE TP rank."""
if loaded_weight.shape[dim] == dst.shape[dim]:
return loaded_weight
rank = get_parallel().moe_tp_rank
return loaded_weight.narrow(dim, rank * dst.shape[dim], dst.shape[dim])
return loaded_weight.narrow(dim, tp_rank * dst.shape[dim], dst.shape[dim])
KV_REPLICATED_SUFFIXES = (
@@ -1238,12 +1229,10 @@ class InklingForConditionalGeneration(nn.Module):
return False
param = params_dict[name]
if loaded_weight.shape != param.shape:
# shared experts shard over the full tp group; routed over moe_tp
tp_rank = (
get_parallel().tp_rank
if ".shared_experts" in name
else get_parallel().moe_tp_rank
)
# Both routed and shared modules own their constructed TP layout.
module = self.get_submodule(name.rsplit(".", 1)[0])
module = getattr(module, "base_layer", module)
tp_rank = module.moe_tp_rank
for dim in range(loaded_weight.ndim):
if loaded_weight.shape[dim] == param.shape[dim]:
continue
@@ -1297,10 +1286,8 @@ class InklingForConditionalGeneration(nn.Module):
hold only its contiguous slice (the fused loader shards the intermediate dim only).
No-op when EP is off or for replicated shared-expert tensors.
"""
ep_size = get_parallel().moe_ep_size
if (
ep_size <= 1
or ".experts." not in name
".experts." not in name
# per-expert RL sync tensors do their own EP remap in _load_per_expert_param;
# a full per-expert tensor whose dim 0 happens to equal n_routed_experts must
# not be pre-narrowed here.
@@ -1309,8 +1296,17 @@ class InklingForConditionalGeneration(nn.Module):
or loaded_weight.shape[0] != self.text_config.n_routed_experts
):
return loaded_weight
local = self.text_config.n_routed_experts // ep_size
start = get_parallel().moe_ep_rank * local
prefix = name.split(".experts.", 1)[0] + ".experts"
try:
module = self.get_submodule(prefix)
except AttributeError:
# Checkpoints may contain a layer absent from this model.
return loaded_weight
module = getattr(module, "base_layer", module)
if module.moe_ep_size <= 1:
return loaded_weight
local = self.text_config.n_routed_experts // module.moe_ep_size
start = module.moe_ep_rank * local
return loaded_weight.narrow(0, start, local).contiguous()
def _load_fused_moe_param(
@@ -1403,10 +1399,10 @@ class InklingForConditionalGeneration(nn.Module):
f"per-expert RL weight-sync does not support the trtllm MoE layout ({target}); "
"serve RL rollouts with the triton MoE runner"
)
ep_size = get_parallel().moe_ep_size
ep_size = moe.moe_ep_size
if ep_size > 1:
local = self.text_config.n_routed_experts // ep_size
first = get_parallel().moe_ep_rank * local
first = moe.moe_ep_rank * local
if not (first <= eid < first + local):
loaded_params.add(target) # another rank owns this expert
return True
@@ -1415,7 +1411,9 @@ class InklingForConditionalGeneration(nn.Module):
dst = params_dict[target].data[
eid
] # [H, I_local]; shard intermediate (dim 1)
dst.copy_(_shard_full_to_local(loaded_weight, dst, dim=1))
dst.copy_(
_shard_full_to_local(loaded_weight, dst, dim=1, tp_rank=moe.moe_tp_rank)
)
else:
w13 = params_dict[target].data[
eid
@@ -1429,7 +1427,9 @@ class InklingForConditionalGeneration(nn.Module):
dst = w13[idx * half : (idx + 1) * half] # contiguous [gate || up]
else:
dst = w13[idx::2] # Inkling-interleaved rows
dst.copy_(_shard_full_to_local(loaded_weight, dst, dim=0))
dst.copy_(
_shard_full_to_local(loaded_weight, dst, dim=0, tp_rank=moe.moe_tp_rank)
)
loaded_params.add(target)
return True
@@ -1590,18 +1590,21 @@ class InklingForConditionalGeneration(nn.Module):
continue
if ".experts.w13_weight" in name:
experts = modules_dict.get(name.rsplit(".", 1)[0])
experts = getattr(experts, "base_layer", experts)
# bf16 routed layers run the stock FusedMoE forward (not moe_tp_forward)
# under --enable-lora, or natively on trtllm_routed for UNQUANTIZED
# checkpoints: de-interleave per moe_tp block for the stock weight prep.
if (
loaded_weight.dtype != torch.uint8
experts is not None
and loaded_weight.dtype != torch.uint8
and self.text_config.inference_moe_w13_interleaved
and (
lora_compatible_layout_enabled()
or bf16_routed_uses_stock_fused_moe(self.quant_config)
)
):
tp = get_parallel().moe_tp_size
tp = experts.moe_tp_size
n_e, two_f, hid = loaded_weight.shape
loaded_weight = deinterleave_gate_up(
loaded_weight.view(n_e, tp, two_f // tp, hid), dim=2
@@ -1625,16 +1628,19 @@ class InklingForConditionalGeneration(nn.Module):
):
continue
if ".shared_experts.shared_w13_weight" in name:
experts = modules_dict.get(name.rsplit(".", 1)[0])
experts = getattr(experts, "base_layer", experts)
# InklingSharedFusedMoE's bf16 path needs contiguous [gate||up] w13 for the
# SRT runner's silu_and_mul, unlike the interleaved bmm/moe_tp_forward paths.
# Per-rank blocks are sized by the FULL tp group (InklingSharedFusedMoE always
# shards over it at EP=1), NOT moe_tp (= tp/ep, wrong under --ep-size > 1).
if (
loaded_weight.dtype != torch.uint8
experts is not None
and loaded_weight.dtype != torch.uint8
and self.text_config.inference_moe_w13_interleaved
and use_inkling_shared_fused_moe()
):
tp = get_parallel().tp_size
tp = experts.moe_tp_size
n_e, two_f, hid = loaded_weight.shape
loaded_weight = deinterleave_gate_up(
loaded_weight.view(n_e, tp, two_f // tp, hid), dim=2
+1 -1
View File
@@ -1739,7 +1739,7 @@ class KimiK3DeltaAttention(nn.Module):
def _a_log_weight_loader(
param: torch.Tensor, loaded_weight: torch.Tensor
) -> None:
tp_rank = get_parallel().attn_tp_rank
tp_rank = self.attn_tp_rank
shard_size = param.data.shape[2] # local_num_heads
start_idx = tp_rank * shard_size
+4 -6
View File
@@ -173,6 +173,7 @@ class MultiHeadRMSNorm(nn.Module):
) -> None:
super().__init__()
self.tp_world = get_parallel().attn_tp_size
self.tp_rank = get_parallel().attn_tp_rank
self.num_heads = num_heads
self.num_heads_per_tp = num_heads // self.tp_world
self.head_dim = head_dim
@@ -183,16 +184,13 @@ class MultiHeadRMSNorm(nn.Module):
self.apply_layernorm_1p = apply_layernorm_1p
self.variance_epsilon = eps
@staticmethod
def weight_loader(
self,
param: nn.Parameter,
loaded_weight: torch.Tensor,
) -> None:
tp_world = get_parallel().attn_tp_size
tp_rank = get_parallel().attn_tp_rank
shard_size = loaded_weight.shape[0] // tp_world
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
shard_size = loaded_weight.shape[0] // self.tp_world
shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size)
param.data.copy_(loaded_weight[shard].reshape_as(param))
def forward(