[Refactor] Load auxiliary and expert checkpoints using owner partitions (#42589)
This commit is contained in:
@@ -431,6 +431,7 @@ class ScaledActivation(nn.Module):
|
||||
self.act = act_module
|
||||
self.input_is_parallel = input_is_parallel
|
||||
if input_is_parallel:
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
tp_size = get_parallel().tp_size
|
||||
intermediate_size_per_partition = divide(intermediate_size, tp_size)
|
||||
else:
|
||||
@@ -448,9 +449,8 @@ class ScaledActivation(nn.Module):
|
||||
def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor):
|
||||
param_data = param.data
|
||||
if self.input_is_parallel:
|
||||
tp_rank = get_parallel().tp_rank
|
||||
shard_size = param_data.shape[0]
|
||||
start_idx = tp_rank * shard_size
|
||||
start_idx = self.tp_rank * shard_size
|
||||
loaded_weight = loaded_weight.narrow(0, start_idx, shard_size)
|
||||
assert param_data.shape == loaded_weight.shape
|
||||
param_data.copy_(loaded_weight)
|
||||
|
||||
+1
-2
@@ -26,7 +26,6 @@ from sglang.srt.layers.quantization.utils import (
|
||||
per_tensor_dequantize,
|
||||
swap_w13_to_w31,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip, set_weight_attrs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -98,7 +97,7 @@ class CompressedTensorsW8A8Fp8MoE(CompressedTensorsMoEScheme):
|
||||
if self.block_quant:
|
||||
assert self.weight_block_size is not None
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_size = layer.moe_tp_size
|
||||
block_n, block_k = (
|
||||
self.weight_block_size[0],
|
||||
self.weight_block_size[1],
|
||||
|
||||
@@ -5,7 +5,6 @@ import re
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
@@ -61,33 +60,12 @@ def _load_gptoss_quark_expert_weights(model, weights, quark_expert_pat):
|
||||
loaded_params: set[str] = set()
|
||||
mxfp4_block = 32
|
||||
|
||||
moe_tp_rank = get_parallel().moe_tp_rank
|
||||
moe_tp_size = get_parallel().moe_tp_size
|
||||
moe_ep_rank = get_parallel().moe_ep_rank
|
||||
moe_ep_size = get_parallel().moe_ep_size
|
||||
|
||||
intermediate_size = model.config.intermediate_size
|
||||
assert intermediate_size % mxfp4_block == 0, (
|
||||
f"{intermediate_size=} must be divisible by {mxfp4_block=}"
|
||||
)
|
||||
intermediate_size_block = intermediate_size // mxfp4_block
|
||||
|
||||
per_rank_intermediate_size_block = math.ceil(intermediate_size_block / moe_tp_size)
|
||||
|
||||
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
|
||||
|
||||
# Calculate common slicing bounds for current rank
|
||||
assert model.config.num_local_experts % moe_ep_size == 0
|
||||
moe_num_local_experts = model.config.num_local_experts // moe_ep_size
|
||||
|
||||
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
|
||||
moe_tp_rank_end = min(
|
||||
(moe_tp_rank + 1) * per_rank_intermediate_size, intermediate_size
|
||||
)
|
||||
|
||||
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
|
||||
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
|
||||
|
||||
for name, weight in weights:
|
||||
# Quark stores experts separately as
|
||||
# `experts.{N}.{gate_up_proj|down_proj}.{suffix}`; pull the
|
||||
@@ -96,6 +74,30 @@ def _load_gptoss_quark_expert_weights(model, weights, quark_expert_pat):
|
||||
if m is None:
|
||||
continue
|
||||
prefix, expert_str, proj, suffix = m.groups()
|
||||
experts = model.get_submodule(prefix)
|
||||
moe_tp_rank = experts.moe_tp_rank
|
||||
moe_tp_size = experts.moe_tp_size
|
||||
moe_ep_rank = experts.moe_ep_rank
|
||||
moe_ep_size = experts.moe_ep_size
|
||||
|
||||
per_rank_intermediate_size_block = math.ceil(
|
||||
intermediate_size_block / moe_tp_size
|
||||
)
|
||||
|
||||
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
|
||||
|
||||
# Calculate common slicing bounds for current rank
|
||||
assert model.config.num_local_experts % moe_ep_size == 0
|
||||
moe_num_local_experts = model.config.num_local_experts // moe_ep_size
|
||||
|
||||
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
|
||||
moe_tp_rank_end = min(
|
||||
(moe_tp_rank + 1) * per_rank_intermediate_size, intermediate_size
|
||||
)
|
||||
|
||||
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
|
||||
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
|
||||
|
||||
global_expert_id = int(expert_str)
|
||||
if global_expert_id < moe_ep_rank_start or global_expert_id >= moe_ep_rank_end:
|
||||
continue
|
||||
|
||||
@@ -386,17 +386,17 @@ class BailingGroupRMSNormGate(RMSNormGated):
|
||||
dtype=dtype,
|
||||
activation="sigmoid",
|
||||
)
|
||||
self.tp_size = get_parallel().attn_tp_size
|
||||
self.tp_rank = get_parallel().attn_tp_rank
|
||||
self.weight.weight_loader = self.weight_loader
|
||||
|
||||
@staticmethod
|
||||
def weight_loader(
|
||||
self,
|
||||
param: torch.nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
) -> None:
|
||||
tp_size = get_parallel().attn_tp_size
|
||||
tp_rank = get_parallel().attn_tp_rank
|
||||
shard_size = loaded_weight.shape[0] // tp_size
|
||||
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
|
||||
shard_size = loaded_weight.shape[0] // self.tp_size
|
||||
shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size)
|
||||
param.data.copy_(loaded_weight[shard].contiguous())
|
||||
return
|
||||
|
||||
|
||||
@@ -84,6 +84,7 @@ class LayerNorm(nn.Module):
|
||||
def __init__(self, param_shape=None, eps=1e-5):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(param_shape))
|
||||
self.tp_rank = get_parallel().tp_rank if self.weight.dim() != 1 else 0
|
||||
self.variance_epsilon = eps
|
||||
set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
|
||||
|
||||
@@ -94,12 +95,11 @@ class LayerNorm(nn.Module):
|
||||
return hidden_states, residuals
|
||||
|
||||
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
|
||||
tp_rank = get_parallel().tp_rank
|
||||
shard_dim = 0 if param.dim() != 1 else None
|
||||
param_data = param.data
|
||||
if shard_dim is not None:
|
||||
shard_size = param_data.shape[shard_dim]
|
||||
start_idx = tp_rank * shard_size
|
||||
start_idx = self.tp_rank * shard_size
|
||||
loaded_weight = loaded_weight.narrow(shard_dim, start_idx, shard_size)
|
||||
assert param_data.shape == loaded_weight.shape
|
||||
param_data.copy_(loaded_weight)
|
||||
|
||||
@@ -103,6 +103,7 @@ class DbrxExperts(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
self.num_total_experts = config.ffn_config.moe_num_experts
|
||||
self.top_k = config.ffn_config.moe_top_k
|
||||
self.d_model = config.d_model
|
||||
@@ -155,10 +156,9 @@ class DbrxExperts(nn.Module):
|
||||
def weight_loader(
|
||||
self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str
|
||||
):
|
||||
tp_rank = get_parallel().tp_rank
|
||||
param_data = param.data
|
||||
shard_size = self.intermediate_size
|
||||
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
|
||||
shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size)
|
||||
# DBRX uses GLU for each experts.
|
||||
# GLU has 3 linear layers: w1, v1 and w2.
|
||||
if weight_name.endswith("w1"):
|
||||
|
||||
@@ -942,11 +942,6 @@ class GptOssForCausalLM(nn.Module):
|
||||
loaded_params: set[str] = set()
|
||||
mxfp4_block = 32
|
||||
|
||||
moe_tp_rank = get_parallel().moe_tp_rank
|
||||
moe_tp_size = get_parallel().moe_tp_size
|
||||
moe_ep_rank = get_parallel().moe_ep_rank
|
||||
moe_ep_size = get_parallel().moe_ep_size
|
||||
|
||||
intermediate_size = self.config.intermediate_size
|
||||
original_intermediate_size = getattr(
|
||||
self.config, "original_intermediate_size", intermediate_size
|
||||
@@ -956,30 +951,49 @@ class GptOssForCausalLM(nn.Module):
|
||||
)
|
||||
intermediate_size_block = intermediate_size // mxfp4_block
|
||||
|
||||
per_rank_intermediate_size_block = math.ceil(
|
||||
intermediate_size_block / moe_tp_size
|
||||
)
|
||||
|
||||
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
|
||||
|
||||
# Calculate common slicing bounds for current rank
|
||||
assert self.config.num_local_experts % moe_ep_size == 0
|
||||
moe_num_global_experts = self.config.num_local_experts
|
||||
moe_num_local_experts = self.config.num_local_experts // moe_ep_size
|
||||
|
||||
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
|
||||
moe_tp_rank_end = min(
|
||||
(moe_tp_rank + 1) * per_rank_intermediate_size, original_intermediate_size
|
||||
)
|
||||
|
||||
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
|
||||
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
|
||||
|
||||
weight_device = next(iter(params_dict.values())).device
|
||||
|
||||
for name, weight in weights:
|
||||
weight = weight.to(weight_device)
|
||||
|
||||
if not any(
|
||||
suffix in name
|
||||
for suffix in (
|
||||
"gate_up_proj_blocks",
|
||||
"down_proj_blocks",
|
||||
"gate_up_proj_scales",
|
||||
"down_proj_scales",
|
||||
"gate_up_proj_bias",
|
||||
"down_proj_bias",
|
||||
)
|
||||
):
|
||||
continue
|
||||
experts = self.get_submodule(name.rsplit(".", 1)[0])
|
||||
moe_tp_rank = experts.moe_tp_rank
|
||||
moe_tp_size = experts.moe_tp_size
|
||||
moe_ep_rank = experts.moe_ep_rank
|
||||
moe_ep_size = experts.moe_ep_size
|
||||
|
||||
per_rank_intermediate_size_block = math.ceil(
|
||||
intermediate_size_block / moe_tp_size
|
||||
)
|
||||
|
||||
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
|
||||
|
||||
# Calculate common slicing bounds for current rank
|
||||
assert self.config.num_local_experts % moe_ep_size == 0
|
||||
moe_num_global_experts = self.config.num_local_experts
|
||||
moe_num_local_experts = self.config.num_local_experts // moe_ep_size
|
||||
|
||||
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
|
||||
moe_tp_rank_end = min(
|
||||
(moe_tp_rank + 1) * per_rank_intermediate_size,
|
||||
original_intermediate_size,
|
||||
)
|
||||
|
||||
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
|
||||
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
|
||||
|
||||
if "gate_up_proj_blocks" in name:
|
||||
# Handle MLP gate and up projection weights
|
||||
new_name = name.replace("gate_up_proj_blocks", "w13_weight")
|
||||
@@ -1240,8 +1254,10 @@ class GptOssForCausalLM(nn.Module):
|
||||
weight_loader = param.weight_loader
|
||||
if "bias" not in name:
|
||||
loaded_weight = loaded_weight.transpose(-2, -1)
|
||||
if "w2_weight_bias" in name and get_parallel().moe_tp_rank != 0:
|
||||
loaded_weight = loaded_weight.zero_()
|
||||
if "w2_weight_bias" in name:
|
||||
experts = self.get_submodule(name.rsplit(".", 1)[0])
|
||||
if experts.moe_tp_rank != 0:
|
||||
loaded_weight = loaded_weight.zero_()
|
||||
|
||||
weight_loader(
|
||||
param,
|
||||
|
||||
@@ -100,29 +100,20 @@ STACKED_DENSE_PARAMS_MAPPING = [
|
||||
("down_proj", "down_proj", None),
|
||||
]
|
||||
|
||||
# Online RL weight-sync streams routed experts one at a time as FULL (unsharded)
|
||||
# per-expert tensors named `...mlp.experts.{j}.gate_proj/up_proj/down_proj.weight`.
|
||||
# Disk checkpoints only ever carry the fused w13_weight/w2_weight, so this pattern
|
||||
# never fires on the ordinary loading path.
|
||||
# Online RL updates use full per-expert gate/up/down tensors; disk checkpoints
|
||||
# contain fused w13/w2 weights and do not match this pattern.
|
||||
_PER_EXPERT_WEIGHT_RE = re.compile(
|
||||
r"^(?P<pfx>.+\.mlp\.experts)\.(?P<eid>\d+)\.(?P<proj>gate_proj|up_proj|down_proj)\.weight$"
|
||||
)
|
||||
|
||||
|
||||
def _shard_full_to_local(
|
||||
loaded_weight: torch.Tensor, dst: torch.Tensor, dim: int
|
||||
loaded_weight: torch.Tensor, dst: torch.Tensor, dim: int, *, tp_rank: int
|
||||
) -> torch.Tensor:
|
||||
"""Slice a FULL (unsharded) per-expert weight to this MoE-TP rank's shard along `dim`.
|
||||
|
||||
The online weight-sync ships full per-expert tensors (parallelism-agnostic HF
|
||||
layout); sglang owns its own MoE-TP sharding, so narrow here per
|
||||
get_parallel().moe_tp_rank. With TP1 the dims already match and this is the
|
||||
identity, so the ordinary path is byte-for-byte unchanged.
|
||||
"""
|
||||
"""Shard full HF per-expert updates with the owning module's MoE TP rank."""
|
||||
if loaded_weight.shape[dim] == dst.shape[dim]:
|
||||
return loaded_weight
|
||||
rank = get_parallel().moe_tp_rank
|
||||
return loaded_weight.narrow(dim, rank * dst.shape[dim], dst.shape[dim])
|
||||
return loaded_weight.narrow(dim, tp_rank * dst.shape[dim], dst.shape[dim])
|
||||
|
||||
|
||||
KV_REPLICATED_SUFFIXES = (
|
||||
@@ -1238,12 +1229,10 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
return False
|
||||
param = params_dict[name]
|
||||
if loaded_weight.shape != param.shape:
|
||||
# shared experts shard over the full tp group; routed over moe_tp
|
||||
tp_rank = (
|
||||
get_parallel().tp_rank
|
||||
if ".shared_experts" in name
|
||||
else get_parallel().moe_tp_rank
|
||||
)
|
||||
# Both routed and shared modules own their constructed TP layout.
|
||||
module = self.get_submodule(name.rsplit(".", 1)[0])
|
||||
module = getattr(module, "base_layer", module)
|
||||
tp_rank = module.moe_tp_rank
|
||||
for dim in range(loaded_weight.ndim):
|
||||
if loaded_weight.shape[dim] == param.shape[dim]:
|
||||
continue
|
||||
@@ -1297,10 +1286,8 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
hold only its contiguous slice (the fused loader shards the intermediate dim only).
|
||||
No-op when EP is off or for replicated shared-expert tensors.
|
||||
"""
|
||||
ep_size = get_parallel().moe_ep_size
|
||||
if (
|
||||
ep_size <= 1
|
||||
or ".experts." not in name
|
||||
".experts." not in name
|
||||
# per-expert RL sync tensors do their own EP remap in _load_per_expert_param;
|
||||
# a full per-expert tensor whose dim 0 happens to equal n_routed_experts must
|
||||
# not be pre-narrowed here.
|
||||
@@ -1309,8 +1296,17 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
or loaded_weight.shape[0] != self.text_config.n_routed_experts
|
||||
):
|
||||
return loaded_weight
|
||||
local = self.text_config.n_routed_experts // ep_size
|
||||
start = get_parallel().moe_ep_rank * local
|
||||
prefix = name.split(".experts.", 1)[0] + ".experts"
|
||||
try:
|
||||
module = self.get_submodule(prefix)
|
||||
except AttributeError:
|
||||
# Checkpoints may contain a layer absent from this model.
|
||||
return loaded_weight
|
||||
module = getattr(module, "base_layer", module)
|
||||
if module.moe_ep_size <= 1:
|
||||
return loaded_weight
|
||||
local = self.text_config.n_routed_experts // module.moe_ep_size
|
||||
start = module.moe_ep_rank * local
|
||||
return loaded_weight.narrow(0, start, local).contiguous()
|
||||
|
||||
def _load_fused_moe_param(
|
||||
@@ -1403,10 +1399,10 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
f"per-expert RL weight-sync does not support the trtllm MoE layout ({target}); "
|
||||
"serve RL rollouts with the triton MoE runner"
|
||||
)
|
||||
ep_size = get_parallel().moe_ep_size
|
||||
ep_size = moe.moe_ep_size
|
||||
if ep_size > 1:
|
||||
local = self.text_config.n_routed_experts // ep_size
|
||||
first = get_parallel().moe_ep_rank * local
|
||||
first = moe.moe_ep_rank * local
|
||||
if not (first <= eid < first + local):
|
||||
loaded_params.add(target) # another rank owns this expert
|
||||
return True
|
||||
@@ -1415,7 +1411,9 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
dst = params_dict[target].data[
|
||||
eid
|
||||
] # [H, I_local]; shard intermediate (dim 1)
|
||||
dst.copy_(_shard_full_to_local(loaded_weight, dst, dim=1))
|
||||
dst.copy_(
|
||||
_shard_full_to_local(loaded_weight, dst, dim=1, tp_rank=moe.moe_tp_rank)
|
||||
)
|
||||
else:
|
||||
w13 = params_dict[target].data[
|
||||
eid
|
||||
@@ -1429,7 +1427,9 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
dst = w13[idx * half : (idx + 1) * half] # contiguous [gate || up]
|
||||
else:
|
||||
dst = w13[idx::2] # Inkling-interleaved rows
|
||||
dst.copy_(_shard_full_to_local(loaded_weight, dst, dim=0))
|
||||
dst.copy_(
|
||||
_shard_full_to_local(loaded_weight, dst, dim=0, tp_rank=moe.moe_tp_rank)
|
||||
)
|
||||
loaded_params.add(target)
|
||||
return True
|
||||
|
||||
@@ -1590,18 +1590,21 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
continue
|
||||
|
||||
if ".experts.w13_weight" in name:
|
||||
experts = modules_dict.get(name.rsplit(".", 1)[0])
|
||||
experts = getattr(experts, "base_layer", experts)
|
||||
# bf16 routed layers run the stock FusedMoE forward (not moe_tp_forward)
|
||||
# under --enable-lora, or natively on trtllm_routed for UNQUANTIZED
|
||||
# checkpoints: de-interleave per moe_tp block for the stock weight prep.
|
||||
if (
|
||||
loaded_weight.dtype != torch.uint8
|
||||
experts is not None
|
||||
and loaded_weight.dtype != torch.uint8
|
||||
and self.text_config.inference_moe_w13_interleaved
|
||||
and (
|
||||
lora_compatible_layout_enabled()
|
||||
or bf16_routed_uses_stock_fused_moe(self.quant_config)
|
||||
)
|
||||
):
|
||||
tp = get_parallel().moe_tp_size
|
||||
tp = experts.moe_tp_size
|
||||
n_e, two_f, hid = loaded_weight.shape
|
||||
loaded_weight = deinterleave_gate_up(
|
||||
loaded_weight.view(n_e, tp, two_f // tp, hid), dim=2
|
||||
@@ -1625,16 +1628,19 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
):
|
||||
continue
|
||||
if ".shared_experts.shared_w13_weight" in name:
|
||||
experts = modules_dict.get(name.rsplit(".", 1)[0])
|
||||
experts = getattr(experts, "base_layer", experts)
|
||||
# InklingSharedFusedMoE's bf16 path needs contiguous [gate||up] w13 for the
|
||||
# SRT runner's silu_and_mul, unlike the interleaved bmm/moe_tp_forward paths.
|
||||
# Per-rank blocks are sized by the FULL tp group (InklingSharedFusedMoE always
|
||||
# shards over it at EP=1), NOT moe_tp (= tp/ep, wrong under --ep-size > 1).
|
||||
if (
|
||||
loaded_weight.dtype != torch.uint8
|
||||
experts is not None
|
||||
and loaded_weight.dtype != torch.uint8
|
||||
and self.text_config.inference_moe_w13_interleaved
|
||||
and use_inkling_shared_fused_moe()
|
||||
):
|
||||
tp = get_parallel().tp_size
|
||||
tp = experts.moe_tp_size
|
||||
n_e, two_f, hid = loaded_weight.shape
|
||||
loaded_weight = deinterleave_gate_up(
|
||||
loaded_weight.view(n_e, tp, two_f // tp, hid), dim=2
|
||||
|
||||
@@ -1739,7 +1739,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
def _a_log_weight_loader(
|
||||
param: torch.Tensor, loaded_weight: torch.Tensor
|
||||
) -> None:
|
||||
tp_rank = get_parallel().attn_tp_rank
|
||||
tp_rank = self.attn_tp_rank
|
||||
shard_size = param.data.shape[2] # local_num_heads
|
||||
start_idx = tp_rank * shard_size
|
||||
|
||||
|
||||
@@ -173,6 +173,7 @@ class MultiHeadRMSNorm(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_world = get_parallel().attn_tp_size
|
||||
self.tp_rank = get_parallel().attn_tp_rank
|
||||
self.num_heads = num_heads
|
||||
self.num_heads_per_tp = num_heads // self.tp_world
|
||||
self.head_dim = head_dim
|
||||
@@ -183,16 +184,13 @@ class MultiHeadRMSNorm(nn.Module):
|
||||
self.apply_layernorm_1p = apply_layernorm_1p
|
||||
self.variance_epsilon = eps
|
||||
|
||||
@staticmethod
|
||||
def weight_loader(
|
||||
self,
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
) -> None:
|
||||
tp_world = get_parallel().attn_tp_size
|
||||
tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
shard_size = loaded_weight.shape[0] // tp_world
|
||||
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
|
||||
shard_size = loaded_weight.shape[0] // self.tp_world
|
||||
shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size)
|
||||
param.data.copy_(loaded_weight[shard].reshape_as(param))
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_cpu_ci
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import torch
|
||||
|
||||
@@ -31,6 +31,7 @@ class _RecordingLayer:
|
||||
|
||||
def __init__(self, is_gated: bool):
|
||||
self.moe_runner_config = MoeRunnerConfig(is_gated=is_gated)
|
||||
self.moe_tp_size = 1
|
||||
self.params = {}
|
||||
|
||||
def register_parameter(self, name, param):
|
||||
@@ -47,20 +48,18 @@ def _create_weights(is_gated: bool, block_quant: bool):
|
||||
is_checkpoint_fp8_serialized=False,
|
||||
)
|
||||
|
||||
with patch.object(fp8_quant, "get_parallel") as parallel:
|
||||
parallel.return_value.tp_size = 1
|
||||
fp8_quant.Fp8MoEMethod.create_fp8_moe_weight_(
|
||||
layer=layer,
|
||||
num_experts=NUM_EXPERTS,
|
||||
hidden_size=HIDDEN,
|
||||
intermediate_size_per_partition=INTERMEDIATE,
|
||||
block_quant=block_quant,
|
||||
quant_config=quant_config,
|
||||
use_mxfp8=False,
|
||||
is_checkpoint_fp8_serialized=False,
|
||||
is_fp4_expert=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
fp8_quant.Fp8MoEMethod.create_fp8_moe_weight_(
|
||||
layer=layer,
|
||||
num_experts=NUM_EXPERTS,
|
||||
hidden_size=HIDDEN,
|
||||
intermediate_size_per_partition=INTERMEDIATE,
|
||||
block_quant=block_quant,
|
||||
quant_config=quant_config,
|
||||
use_mxfp8=False,
|
||||
is_checkpoint_fp8_serialized=False,
|
||||
is_fp4_expert=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
return layer.params
|
||||
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ import torch
|
||||
|
||||
from sglang.srt.layers.quantization.blockwise_int8 import BlockInt8Config
|
||||
from sglang.srt.layers.quantization.w8a8_int8 import W8A8Int8Config
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_device_sm
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.layer_ut_utils import (
|
||||
@@ -113,9 +112,7 @@ class TestBlockInt8Linear(_Int8LinearCheck):
|
||||
activation_scheme="dynamic",
|
||||
weight_block_size=[128, 128],
|
||||
)
|
||||
# create_weights reads get_parallel().tp_size, not the layer's argument.
|
||||
with get_parallel().override(tp_size=1, tp_rank=0):
|
||||
layer = make_tp1_column_parallel_linear(quant_config, n, k)
|
||||
layer = make_tp1_column_parallel_linear(quant_config, n, k)
|
||||
w = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) / 10
|
||||
w_int8, scale_inv, w_dequant = _quantize_int8_block(w)
|
||||
load_linear_weights(layer, weight=w_int8, weight_scale_inv=scale_inv)
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Auxiliary checkpoint shards stay with the constructed activation/norm layer."""
|
||||
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.activation import ScaledActivation
|
||||
from sglang.srt.models.bailing_moe_linear import BailingGroupRMSNormGate
|
||||
from sglang.srt.models.commandr import LayerNorm
|
||||
from sglang.srt.models.kimi_k3 import KimiK3DeltaAttention
|
||||
from sglang.srt.models.minimax_m3 import MultiHeadRMSNorm
|
||||
from sglang.srt.runtime_context import SpawnRanks, get_parallel, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.parallel_groups import parallel_scope, publish
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large")
|
||||
|
||||
KINDS = (
|
||||
"scaled",
|
||||
"scaled_replicated",
|
||||
"command",
|
||||
"command_replicated",
|
||||
"bailing",
|
||||
"minimax",
|
||||
"minimax_1p",
|
||||
"kimi",
|
||||
)
|
||||
|
||||
|
||||
def loading_scope(changed):
|
||||
if not changed:
|
||||
return nullcontext()
|
||||
return parallel_scope(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
tp_group=None,
|
||||
attn_tp_size=1,
|
||||
attn_tp_rank=0,
|
||||
attn_tp_group=None,
|
||||
attn_dp_size=1,
|
||||
attn_dp_rank=0,
|
||||
attn_cp_size=1,
|
||||
attn_cp_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_ep_group=None,
|
||||
moe_dp_size=1,
|
||||
moe_dp_rank=0,
|
||||
)
|
||||
|
||||
|
||||
def build_layer(kind):
|
||||
parallel = get_parallel()
|
||||
if kind.startswith("scaled"):
|
||||
replicated = kind.endswith("replicated")
|
||||
module = ScaledActivation(nn.GELU(), 32, input_is_parallel=not replicated)
|
||||
return (
|
||||
module,
|
||||
module.scales,
|
||||
(32,),
|
||||
0 if replicated else parallel.tp_rank,
|
||||
1 if replicated else parallel.tp_size,
|
||||
)
|
||||
if kind.startswith("command"):
|
||||
replicated = kind.endswith("replicated")
|
||||
module = LayerNorm(16 if replicated else (8 // parallel.tp_size, 16))
|
||||
return (
|
||||
module,
|
||||
module.weight,
|
||||
(16,) if replicated else (8, 16),
|
||||
0 if replicated else parallel.tp_rank,
|
||||
1 if replicated else parallel.tp_size,
|
||||
)
|
||||
if kind == "bailing":
|
||||
module = BailingGroupRMSNormGate(64 // parallel.attn_tp_size, group_size=8)
|
||||
return (
|
||||
module,
|
||||
module.weight,
|
||||
(64,),
|
||||
parallel.attn_tp_rank,
|
||||
parallel.attn_tp_size,
|
||||
)
|
||||
if kind.startswith("minimax"):
|
||||
module = MultiHeadRMSNorm(8, 16, apply_layernorm_1p=kind.endswith("1p"))
|
||||
return (
|
||||
module,
|
||||
module.weight,
|
||||
(8, 16),
|
||||
parallel.attn_tp_rank,
|
||||
parallel.attn_tp_size,
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
dtype=torch.get_default_dtype(),
|
||||
v_head_dim=8,
|
||||
rms_norm_eps=1e-6,
|
||||
linear_attn_config=dict(head_dim=8, num_heads=8, short_conv_kernel_size=3),
|
||||
)
|
||||
module = KimiK3DeltaAttention(0, 32, config)
|
||||
return module, module.A_log, (8,), parallel.attn_tp_rank, parallel.attn_tp_size
|
||||
|
||||
|
||||
def values(shape, param, offset=0):
|
||||
count = 1
|
||||
for size in shape:
|
||||
count *= size
|
||||
return ((torch.arange(count, device=param.device) + offset) % 17 + 1).reshape(
|
||||
shape
|
||||
).to(param.dtype) / 32
|
||||
|
||||
|
||||
def load_layer(kind, *, changed=False, flat=False, old_kimi=False, offset=0):
|
||||
module, param, shape, rank, size = build_layer(kind)
|
||||
full = values(shape, param, offset)
|
||||
checkpoint = full.flatten() if flat else full
|
||||
if old_kimi:
|
||||
checkpoint = full.view(1, 1, 8, 1)
|
||||
expected = full.chunk(size, dim=0)[rank].reshape_as(param)
|
||||
with loading_scope(changed):
|
||||
param.weight_loader(param, checkpoint)
|
||||
torch.testing.assert_close(param, expected, rtol=0, atol=0)
|
||||
return module, param, expected
|
||||
|
||||
|
||||
class TestAuxiliaryLoaderLayouts(CustomTestCase):
|
||||
def setUp(self):
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
|
||||
def check_loads(self, changed):
|
||||
for dp in (1, 2):
|
||||
for rank in range(4):
|
||||
reset_context()
|
||||
publish(
|
||||
ServerArgs(
|
||||
model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp
|
||||
),
|
||||
role="test",
|
||||
ranks=SpawnRanks(world_rank=rank),
|
||||
)
|
||||
for kind in KINDS:
|
||||
for offset in (0, 7):
|
||||
load_layer(kind, changed=changed, offset=offset)
|
||||
if kind.startswith("minimax"):
|
||||
load_layer(kind, changed=changed, flat=True, offset=offset)
|
||||
if kind == "kimi":
|
||||
load_layer(
|
||||
kind, changed=changed, old_kimi=True, offset=offset
|
||||
)
|
||||
|
||||
def test_loaders_after_scope_exit(self):
|
||||
self.check_loads(True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,116 @@
|
||||
"""DBRX expert checkpoint shards follow the constructed TP layout."""
|
||||
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
|
||||
from sglang.srt.models.dbrx import DbrxExperts
|
||||
from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=15, stage="nightly", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
def loading_scope(changed):
|
||||
if not changed:
|
||||
return nullcontext()
|
||||
return get_parallel().override(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
tp_group=None,
|
||||
attn_tp_size=1,
|
||||
attn_tp_rank=0,
|
||||
attn_tp_group=None,
|
||||
attn_dp_size=1,
|
||||
attn_dp_rank=0,
|
||||
attn_cp_size=1,
|
||||
attn_cp_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_ep_group=None,
|
||||
moe_dp_size=1,
|
||||
moe_dp_rank=0,
|
||||
)
|
||||
|
||||
|
||||
def build_experts(dtype=torch.bfloat16):
|
||||
config = SimpleNamespace(
|
||||
d_model=32,
|
||||
ffn_config=SimpleNamespace(moe_num_experts=4, moe_top_k=2, ffn_hidden_size=128),
|
||||
)
|
||||
module = DbrxExperts(config, 0, params_dtype=dtype)
|
||||
assert isinstance(module.router.layer, ReplicatedLinear)
|
||||
assert isinstance(module.router.layer.quant_method, UnquantizedLinearMethod)
|
||||
return module
|
||||
|
||||
|
||||
def values(shape, dtype, offset=0):
|
||||
count = 1
|
||||
for size in shape:
|
||||
count *= size
|
||||
return (
|
||||
(((torch.arange(count, device="cuda") + offset) % 31 - 15) / 128)
|
||||
.to(dtype)
|
||||
.reshape(shape)
|
||||
)
|
||||
|
||||
|
||||
def load_experts(module, *, changed=False, flat=False, offset=0):
|
||||
rank = get_parallel().tp_rank
|
||||
start = rank * module.intermediate_size
|
||||
stop = start + module.intermediate_size
|
||||
loaded = {}
|
||||
for index, name in enumerate(("w1", "v1", "w2")):
|
||||
full = values((4, 128, 32), module.ws.dtype, offset + index * 7)
|
||||
param = module.w2s if name == "w2" else module.ws
|
||||
with loading_scope(changed):
|
||||
param.weight_loader(param, full.flatten() if flat else full, name)
|
||||
loaded[name] = full[:, start:stop, :]
|
||||
expected_ws = torch.cat((loaded["w1"], loaded["v1"]), dim=1)
|
||||
expected_w2 = loaded["w2"].transpose(1, 2)
|
||||
torch.testing.assert_close(module.ws, expected_ws, rtol=0, atol=0)
|
||||
torch.testing.assert_close(module.w2s, expected_w2, rtol=0, atol=0)
|
||||
return expected_ws, expected_w2
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU")
|
||||
class TestDbrxExpertLoaderLayout(CustomTestCase):
|
||||
def setUp(self):
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
|
||||
def check_loads(self, changed):
|
||||
with torch.inference_mode():
|
||||
for size in (1, 2, 4):
|
||||
for rank in range(size):
|
||||
reset_context()
|
||||
publish(
|
||||
ServerArgs(model_path="dummy", device="cuda", tp_size=size),
|
||||
role="test",
|
||||
ranks=SpawnRanks(world_rank=rank),
|
||||
)
|
||||
for dtype in (torch.float32, torch.bfloat16):
|
||||
module = build_experts(dtype)
|
||||
for flat in (False, True):
|
||||
for offset in (0, 11):
|
||||
load_experts(
|
||||
module,
|
||||
changed=changed,
|
||||
flat=flat,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
def test_native_loader_after_scope_exit(self):
|
||||
self.check_loads(True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,344 @@
|
||||
"""GPT-OSS packed expert checkpoints follow the destination expert layout."""
|
||||
|
||||
import re
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
from sglang.srt.layers.quantization.mxfp4 import Mxfp4Config, Mxfp4MoEMethod
|
||||
from sglang.srt.layers.quantization.quark.quark import QuarkConfig
|
||||
from sglang.srt.layers.quantization.quark.schemes.quark_w4a8_mxfp4_moe import (
|
||||
QuarkW4A8MXFp4MoE,
|
||||
)
|
||||
from sglang.srt.layers.quantization.quark.weights import (
|
||||
_load_gptoss_quark_expert_weights,
|
||||
)
|
||||
from sglang.srt.models.gpt_oss import GptOssForCausalLM
|
||||
from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large")
|
||||
|
||||
EXPERT_PATTERN = re.compile(
|
||||
r"^(.*\.mlp\.experts)\.(\d+)\.(gate_up_proj|down_proj)\."
|
||||
r"(weight|weight_scale|input_scale|bias)$"
|
||||
)
|
||||
|
||||
|
||||
def loading_scope(changed):
|
||||
if not changed:
|
||||
return nullcontext()
|
||||
return get_parallel().override(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
tp_group=None,
|
||||
attn_tp_size=1,
|
||||
attn_tp_rank=0,
|
||||
attn_tp_group=None,
|
||||
attn_dp_size=1,
|
||||
attn_dp_rank=0,
|
||||
attn_cp_size=1,
|
||||
attn_cp_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_ep_group=None,
|
||||
moe_dp_size=1,
|
||||
moe_dp_rank=0,
|
||||
)
|
||||
|
||||
|
||||
def quark_config():
|
||||
return QuarkConfig(
|
||||
quant_config={
|
||||
"packed_modules_mapping": {},
|
||||
"layer_quant_config": {},
|
||||
"layer_type_quant_config": {},
|
||||
"global_quant_config": {
|
||||
"weight": {
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"is_dynamic": False,
|
||||
"scale_format": "e8m0",
|
||||
},
|
||||
"input_tensors": {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
"output_tensors": None,
|
||||
"bias": None,
|
||||
},
|
||||
},
|
||||
is_prequantized=True,
|
||||
)
|
||||
|
||||
|
||||
def build_model(kind, intermediate=384, original=None, device="cuda"):
|
||||
config = SimpleNamespace(
|
||||
hidden_size=128,
|
||||
intermediate_size=intermediate,
|
||||
num_local_experts=4,
|
||||
num_hidden_layers=1,
|
||||
)
|
||||
if original is not None:
|
||||
config.original_intermediate_size = original
|
||||
model = GptOssForCausalLM.__new__(GptOssForCausalLM)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.config = config
|
||||
model.quant_config = (
|
||||
Mxfp4Config(is_checkpoint_mxfp4_serialized=True)
|
||||
if kind == "mxfp4"
|
||||
else quark_config()
|
||||
if kind == "quark"
|
||||
else None
|
||||
)
|
||||
# Quark's W4A8 runner is AMD-only. Keep native parameter construction and
|
||||
# the checkpoint loader while avoiding runner creation in this CUDA test.
|
||||
runner_scope = (
|
||||
patch.object(QuarkW4A8MXFp4MoE, "create_moe_runner", return_value=None)
|
||||
if kind == "quark"
|
||||
else nullcontext()
|
||||
)
|
||||
with torch.device(device), runner_scope:
|
||||
experts = FusedMoE(
|
||||
num_experts=4,
|
||||
top_k=2,
|
||||
hidden_size=128,
|
||||
intermediate_size=intermediate,
|
||||
layer_id=0,
|
||||
gemm1_alpha=1.702,
|
||||
gemm1_clamp_limit=10.0,
|
||||
quant_config=model.quant_config,
|
||||
params_dtype=torch.bfloat16,
|
||||
with_bias=True,
|
||||
use_weight_loader_fused=kind == "unquantized",
|
||||
reduce_results=False,
|
||||
prefix="model.layers.0.mlp.experts",
|
||||
)
|
||||
if kind == "mxfp4":
|
||||
assert isinstance(experts.quant_method, Mxfp4MoEMethod)
|
||||
elif kind == "quark":
|
||||
assert isinstance(experts.scheme, QuarkW4A8MXFp4MoE)
|
||||
model.model = torch.nn.Module()
|
||||
model.model.layers = torch.nn.ModuleList([torch.nn.Module()])
|
||||
model.model.layers[0].mlp = torch.nn.Module()
|
||||
model.model.layers[0].mlp.experts = experts
|
||||
return model, experts
|
||||
|
||||
|
||||
def values(shape, dtype, offset=0, device="cuda"):
|
||||
count = 1
|
||||
for size in shape:
|
||||
count *= size
|
||||
indices = torch.arange(count, device=device) + offset
|
||||
data = (
|
||||
(indices % 16 + (indices // 16 % 16) * 16)
|
||||
if dtype == torch.uint8
|
||||
else (indices % 31 - 15) / 64
|
||||
)
|
||||
return data.to(dtype).reshape(shape)
|
||||
|
||||
|
||||
def load_experts(model, kind, *, changed=False, offset=0):
|
||||
module = model.model.layers[0].mlp.experts
|
||||
config = model.config
|
||||
intermediate = getattr(
|
||||
config, "original_intermediate_size", config.intermediate_size
|
||||
)
|
||||
hidden = config.hidden_size
|
||||
device = module.w13_weight.device
|
||||
source = {
|
||||
"w13_weight": values(
|
||||
(4, 2 * intermediate, hidden // 2), torch.uint8, offset, device
|
||||
),
|
||||
"w2_weight": values(
|
||||
(4, hidden, intermediate // 2), torch.uint8, offset + 3, device
|
||||
),
|
||||
"w13_weight_scale": values(
|
||||
(4, 2 * intermediate, hidden // 32), torch.uint8, offset + 5, device
|
||||
),
|
||||
"w2_weight_scale": values(
|
||||
(4, hidden, intermediate // 32), torch.uint8, offset + 7, device
|
||||
),
|
||||
"w13_weight_bias": values(
|
||||
(4, 2 * intermediate), module.w13_weight_bias.dtype, offset + 11, device
|
||||
),
|
||||
"w2_weight_bias": values(
|
||||
(4, hidden), module.w2_weight_bias.dtype, offset + 13, device
|
||||
),
|
||||
}
|
||||
if kind == "quark":
|
||||
source["w13_input_scale"] = values((4,), torch.float32, offset + 17, device)
|
||||
source["w2_input_scale"] = values((4,), torch.float32, offset + 19, device)
|
||||
prefix = "model.layers.0.mlp.experts"
|
||||
suffixes = (
|
||||
("w13_weight", "gate_up_proj", "blocks"),
|
||||
("w2_weight", "down_proj", "blocks"),
|
||||
("w13_weight_scale", "gate_up_proj", "scales"),
|
||||
("w2_weight_scale", "down_proj", "scales"),
|
||||
("w13_weight_bias", "gate_up_proj", "bias"),
|
||||
("w2_weight_bias", "down_proj", "bias"),
|
||||
)
|
||||
checkpoint = []
|
||||
if kind == "mxfp4":
|
||||
for param_name, projection, suffix in suffixes:
|
||||
data = source[param_name]
|
||||
if suffix == "blocks":
|
||||
data = data.unflatten(-1, (-1, 16))
|
||||
checkpoint.append((f"{prefix}.{projection}_{suffix}", data))
|
||||
checkpoint.append((f"{prefix}.unused_tensor", torch.ones(1, device=device)))
|
||||
else:
|
||||
for expert in range(4):
|
||||
for param_name, projection, suffix in suffixes:
|
||||
suffix = {"blocks": "weight", "scales": "weight_scale"}.get(
|
||||
suffix, suffix
|
||||
)
|
||||
checkpoint.append(
|
||||
(
|
||||
f"{prefix}.{expert}.{projection}.{suffix}",
|
||||
source[param_name][expert],
|
||||
)
|
||||
)
|
||||
for param_name, projection in (
|
||||
("w13_input_scale", "gate_up_proj"),
|
||||
("w2_input_scale", "down_proj"),
|
||||
):
|
||||
checkpoint.append(
|
||||
(
|
||||
f"{prefix}.{expert}.{projection}.input_scale",
|
||||
source[param_name][expert],
|
||||
)
|
||||
)
|
||||
checkpoint.append(("not-an-expert-weight", torch.ones(1, device=device)))
|
||||
before = {name: param.detach().clone() for name, param in module.named_parameters()}
|
||||
with loading_scope(changed):
|
||||
loaded = (
|
||||
model._load_mxfp4_experts_weights(iter(checkpoint))
|
||||
if kind == "mxfp4"
|
||||
else _load_gptoss_quark_expert_weights(
|
||||
model, iter(checkpoint), EXPERT_PATTERN
|
||||
)
|
||||
)
|
||||
width = (
|
||||
(config.intermediate_size // 32 + module.moe_tp_size - 1) // module.moe_tp_size
|
||||
) * 32
|
||||
start = module.moe_tp_rank * width
|
||||
stop = min(start + width, intermediate)
|
||||
expert_start = module.moe_ep_rank * module.num_local_experts
|
||||
expert_stop = expert_start + module.num_local_experts
|
||||
expected = {}
|
||||
for name, original_data in source.items():
|
||||
data = original_data[expert_start:expert_stop]
|
||||
target = before[name]
|
||||
if name.startswith("w13") and "input_scale" not in name:
|
||||
if kind == "mxfp4":
|
||||
data = data[:, 2 * start : 2 * stop]
|
||||
if data.dim() == 3:
|
||||
target[:, : data.shape[1], : data.shape[2]] = data
|
||||
else:
|
||||
target[:, : data.shape[1]] = data
|
||||
else:
|
||||
gate = data[:, 0::2, ...][:, start:stop]
|
||||
up = data[:, 1::2, ...][:, start:stop]
|
||||
half = target.shape[1] // 2
|
||||
if data.dim() == 3:
|
||||
target[:, : gate.shape[1], : gate.shape[2]] = gate
|
||||
target[:, half : half + up.shape[1], : up.shape[2]] = up
|
||||
else:
|
||||
target[:, : gate.shape[1]] = gate
|
||||
target[:, half : half + up.shape[1]] = up
|
||||
elif name == "w2_weight":
|
||||
data = data[..., start // 2 : stop // 2]
|
||||
target[:, : data.shape[1], : data.shape[2]] = data
|
||||
elif name == "w2_weight_scale":
|
||||
data = data[..., start // 32 : stop // 32]
|
||||
target[:, : data.shape[1], : data.shape[2]] = data
|
||||
elif name == "w2_weight_bias":
|
||||
target[:, : data.shape[1]] = (
|
||||
data if module.moe_tp_rank == 0 else torch.zeros_like(data)
|
||||
)
|
||||
else:
|
||||
target.copy_(data)
|
||||
torch.testing.assert_close(getattr(module, name), target, rtol=0, atol=0)
|
||||
expected[name] = target
|
||||
assert loaded == {f"{prefix}.{name}" for name in source}
|
||||
return expected
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU")
|
||||
class TestGptOssExpertLoaderLayout(CustomTestCase):
|
||||
def setUp(self):
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
|
||||
def check_loads(self, changed):
|
||||
with torch.inference_mode():
|
||||
for ep_size in (1, 2, 4):
|
||||
for rank in range(4):
|
||||
reset_context()
|
||||
publish(
|
||||
ServerArgs(
|
||||
model_path="dummy",
|
||||
device="cuda",
|
||||
tp_size=4,
|
||||
ep_size=ep_size,
|
||||
moe_runner_backend="triton",
|
||||
),
|
||||
role="test",
|
||||
ranks=SpawnRanks(world_rank=rank),
|
||||
)
|
||||
for kind in ("mxfp4", "quark"):
|
||||
for intermediate, original in (
|
||||
((384, None), (288, None), (384, 288))
|
||||
if kind == "mxfp4"
|
||||
else ((384, None), (512, None))
|
||||
):
|
||||
model, _ = build_model(kind, intermediate, original)
|
||||
for offset in (0, 11):
|
||||
load_experts(
|
||||
model, kind, changed=changed, offset=offset
|
||||
)
|
||||
|
||||
def test_normal_down_projection_bias_uses_the_expert_rank(self):
|
||||
with torch.inference_mode():
|
||||
for rank in range(4):
|
||||
reset_context()
|
||||
publish(
|
||||
ServerArgs(
|
||||
model_path="dummy",
|
||||
device="cuda",
|
||||
tp_size=4,
|
||||
moe_runner_backend="triton",
|
||||
),
|
||||
role="test",
|
||||
ranks=SpawnRanks(world_rank=rank),
|
||||
)
|
||||
model, module = build_model("unquantized")
|
||||
for changed in (False, True):
|
||||
bias = values((4, 128), module.w2_weight_bias.dtype)
|
||||
expected = bias.clone() if rank == 0 else torch.zeros_like(bias)
|
||||
with loading_scope(changed):
|
||||
model._load_normal_weights(
|
||||
[("model.layers.0.mlp.experts.down_proj_bias", bias)],
|
||||
is_nextn=False,
|
||||
weight_name_mapping=None,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
module.w2_weight_bias, expected, rtol=0, atol=0
|
||||
)
|
||||
|
||||
def test_native_loaders_after_scope_exit(self):
|
||||
self.check_loads(True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Inkling checkpoint preprocessing follows its constructed expert modules."""
|
||||
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.lora.layers import BaseLayerWithLoRA
|
||||
from sglang.srt.models import inkling
|
||||
from sglang.srt.models.inkling_common.moe import InklingSharedFusedMoE
|
||||
from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
|
||||
|
||||
|
||||
def values(shape, offset=0, *, device="cpu", dtype=torch.float32):
|
||||
count = 1
|
||||
for size in shape:
|
||||
count *= size
|
||||
return (
|
||||
(((torch.arange(count, device=device) + offset) % 29 - 14) / 128)
|
||||
.reshape(shape)
|
||||
.to(dtype)
|
||||
)
|
||||
|
||||
|
||||
def loading_scope(changed):
|
||||
if not changed:
|
||||
return nullcontext()
|
||||
return get_parallel().override(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
tp_group=None,
|
||||
attn_tp_size=1,
|
||||
attn_tp_rank=0,
|
||||
attn_tp_group=None,
|
||||
attn_dp_size=1,
|
||||
attn_dp_rank=0,
|
||||
attn_cp_size=1,
|
||||
attn_cp_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_ep_group=None,
|
||||
moe_dp_size=1,
|
||||
moe_dp_rank=0,
|
||||
)
|
||||
|
||||
|
||||
def build_model(*, width=32, intermediate=64, wrapped=False):
|
||||
model = inkling.InklingForConditionalGeneration.__new__(
|
||||
inkling.InklingForConditionalGeneration
|
||||
)
|
||||
nn.Module.__init__(model)
|
||||
model.text_config = SimpleNamespace(
|
||||
n_routed_experts=8, inference_moe_w13_interleaved=True
|
||||
)
|
||||
model.quant_config = model.audio = model.visual = None
|
||||
model.llm = nn.Module()
|
||||
block = nn.Module()
|
||||
block.mlp = nn.Module()
|
||||
routed = FusedMoE(
|
||||
num_experts=8,
|
||||
hidden_size=width,
|
||||
intermediate_size=intermediate,
|
||||
layer_id=0,
|
||||
top_k=1,
|
||||
use_weight_loader_fused=True,
|
||||
)
|
||||
shared = InklingSharedFusedMoE(
|
||||
2, width, intermediate, 0, "llm.layers.0.mlp.shared_experts", None, True
|
||||
)
|
||||
block.mlp.experts = BaseLayerWithLoRA(routed, Mock()) if wrapped else routed
|
||||
block.mlp.shared_experts = shared
|
||||
model.llm.layers = nn.ModuleList([block])
|
||||
return model, routed, shared
|
||||
|
||||
|
||||
def load_per_expert(model, routed, *, changed=False, contiguous=False, offset=0):
|
||||
width = routed.hidden_size
|
||||
intermediate = routed.intermediate_size_per_partition * routed.moe_tp_size
|
||||
local = 8 // routed.moe_ep_size
|
||||
start = routed.moe_ep_rank * local
|
||||
shard = routed.moe_tp_rank * routed.intermediate_size_per_partition
|
||||
weights = []
|
||||
gates = []
|
||||
ups = []
|
||||
downs = []
|
||||
for expert in range(8):
|
||||
gate = values(
|
||||
(intermediate, width),
|
||||
offset + 3 * expert,
|
||||
device=routed.w13_weight.device,
|
||||
dtype=routed.w13_weight.dtype,
|
||||
)
|
||||
up = values(
|
||||
(intermediate, width),
|
||||
offset + 3 * expert + 1,
|
||||
device=gate.device,
|
||||
dtype=gate.dtype,
|
||||
)
|
||||
down = values(
|
||||
(width, intermediate),
|
||||
offset + 3 * expert + 2,
|
||||
device=gate.device,
|
||||
dtype=gate.dtype,
|
||||
)
|
||||
for name, weight in (("gate_proj", gate), ("up_proj", up), ("down_proj", down)):
|
||||
weights.append(
|
||||
(f"model.llm.layers.0.mlp.experts.{expert}.{name}.weight", weight)
|
||||
)
|
||||
if start <= expert < start + local:
|
||||
gates.append(gate[shard : shard + routed.intermediate_size_per_partition])
|
||||
ups.append(up[shard : shard + routed.intermediate_size_per_partition])
|
||||
downs.append(
|
||||
down[:, shard : shard + routed.intermediate_size_per_partition]
|
||||
)
|
||||
with (
|
||||
loading_scope(changed),
|
||||
patch.object(
|
||||
inkling, "lora_compatible_layout_enabled", return_value=contiguous
|
||||
),
|
||||
):
|
||||
loaded = model.load_weights(weights)
|
||||
gate, up = torch.stack(gates), torch.stack(ups)
|
||||
expected_w13 = (
|
||||
torch.cat((gate, up), dim=1)
|
||||
if contiguous
|
||||
else torch.stack((gate, up), dim=2).flatten(1, 2)
|
||||
)
|
||||
expected_w2 = torch.stack(downs)
|
||||
suffix = (
|
||||
".base_layer"
|
||||
if isinstance(model.llm.layers[0].mlp.experts, BaseLayerWithLoRA)
|
||||
else ""
|
||||
)
|
||||
assert loaded == {
|
||||
f"llm.layers.0.mlp.experts{suffix}.w13_weight",
|
||||
f"llm.layers.0.mlp.experts{suffix}.w2_weight",
|
||||
}
|
||||
torch.testing.assert_close(routed.w13_weight, expected_w13, rtol=0, atol=0)
|
||||
torch.testing.assert_close(routed.w2_weight, expected_w2, rtol=0, atol=0)
|
||||
return expected_w13, expected_w2
|
||||
|
||||
|
||||
def load_fused(model, routed, shared, *, changed=False, offset=0):
|
||||
weights = []
|
||||
expected = []
|
||||
for kind, module, experts in (
|
||||
("experts", routed, 8),
|
||||
("shared_experts", shared, 2),
|
||||
):
|
||||
width = module.hidden_size
|
||||
intermediate = module.intermediate_size_per_partition * module.moe_tp_size
|
||||
w13 = values(
|
||||
(experts, 2 * intermediate, width),
|
||||
offset,
|
||||
device=module.w13_weight.device,
|
||||
dtype=module.w13_weight.dtype,
|
||||
)
|
||||
w2 = values(
|
||||
(experts, width, intermediate),
|
||||
offset + 1,
|
||||
device=w13.device,
|
||||
dtype=w13.dtype,
|
||||
)
|
||||
prefix = f"model.llm.layers.0.mlp.{kind}."
|
||||
weights.extend(
|
||||
(
|
||||
(
|
||||
prefix
|
||||
+ (
|
||||
"shared_w13_weight"
|
||||
if kind == "shared_experts"
|
||||
else "w13_weight"
|
||||
),
|
||||
w13,
|
||||
),
|
||||
(
|
||||
prefix
|
||||
+ ("shared_w2_weight" if kind == "shared_experts" else "w2_weight"),
|
||||
w2,
|
||||
),
|
||||
)
|
||||
)
|
||||
local = experts // module.moe_ep_size
|
||||
first = module.moe_ep_rank * local
|
||||
rank = module.moe_tp_rank
|
||||
local_w13 = w13[first : first + local].chunk(module.moe_tp_size, dim=1)[rank]
|
||||
local_w13 = torch.cat((local_w13[:, 0::2], local_w13[:, 1::2]), dim=1)
|
||||
local_w2 = w2[first : first + local].chunk(module.moe_tp_size, dim=2)[rank]
|
||||
expected.append((module, local_w13, local_w2))
|
||||
weights.append(("model.llm.layers.999.mlp.experts.w13_weight", weights[0][1]))
|
||||
with (
|
||||
loading_scope(changed),
|
||||
patch.object(inkling, "lora_compatible_layout_enabled", return_value=True),
|
||||
patch.object(inkling, "use_inkling_shared_fused_moe", return_value=True),
|
||||
):
|
||||
loaded = model.load_weights(weights)
|
||||
assert loaded == {
|
||||
f"llm.layers.0.mlp.{kind}.{leaf}"
|
||||
for kind in ("experts", "shared_experts")
|
||||
for leaf in ("w13_weight", "w2_weight")
|
||||
}
|
||||
for module, w13, w2 in expected:
|
||||
torch.testing.assert_close(module.w13_weight, w13, rtol=0, atol=0)
|
||||
torch.testing.assert_close(module.w2_weight, w2, rtol=0, atol=0)
|
||||
return expected
|
||||
|
||||
|
||||
class TestInklingMoeLoaderLayout(CustomTestCase):
|
||||
def setUp(self):
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
|
||||
def check_layouts(self, *, changed=False, wrapped=False):
|
||||
for ep in (1, 2, 4):
|
||||
for rank in range(4):
|
||||
publish(
|
||||
ServerArgs(
|
||||
model_path="dummy",
|
||||
device="cpu",
|
||||
tp_size=4,
|
||||
ep_size=ep,
|
||||
moe_runner_backend="triton",
|
||||
),
|
||||
role="test",
|
||||
ranks=SpawnRanks(world_rank=rank),
|
||||
)
|
||||
for contiguous in (False, True):
|
||||
model, routed, _ = build_model(wrapped=wrapped)
|
||||
for offset in (0, 7):
|
||||
load_per_expert(
|
||||
model,
|
||||
routed,
|
||||
changed=changed,
|
||||
contiguous=contiguous,
|
||||
offset=offset,
|
||||
)
|
||||
if not wrapped:
|
||||
model, routed, shared = build_model()
|
||||
for offset in (0, 7):
|
||||
load_fused(
|
||||
model, routed, shared, changed=changed, offset=offset
|
||||
)
|
||||
|
||||
def test_native_expert_loaders_after_scope_exit(self):
|
||||
self.check_layouts(changed=True)
|
||||
|
||||
def test_native_per_expert_loader_with_lora_parameter_aliases(self):
|
||||
self.check_layouts(changed=True, wrapped=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -20,7 +20,7 @@ import unittest
|
||||
import torch
|
||||
|
||||
import sglang.srt.models.inkling as inkling_mod
|
||||
from sglang.srt.runtime_context import reset_context
|
||||
from sglang.srt.runtime_context import get_parallel, reset_context
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import publish_build_topology
|
||||
|
||||
@@ -38,6 +38,10 @@ class _FakeModel:
|
||||
inference_moe_w13_interleaved=interleaved,
|
||||
)
|
||||
self._moe = moe if moe is not None else types.SimpleNamespace()
|
||||
self._moe.moe_ep_size = get_parallel().moe_ep_size
|
||||
self._moe.moe_ep_rank = get_parallel().moe_ep_rank
|
||||
self._moe.moe_tp_size = get_parallel().moe_tp_size
|
||||
self._moe.moe_tp_rank = get_parallel().moe_tp_rank
|
||||
|
||||
def get_submodule(self, path):
|
||||
return self._moe
|
||||
|
||||
Reference in New Issue
Block a user