From 5e12831cec1c093202cfeecfc2ff3918e8cb6cd9 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:24:03 -0700 Subject: [PATCH] [Refactor] Load auxiliary and expert checkpoints using owner partitions (#42589) --- python/sglang/srt/layers/activation.py | 4 +- .../compressed_tensors_w8a8_fp8_moe.py | 3 +- .../srt/layers/quantization/quark/weights.py | 46 +-- .../sglang/srt/models/bailing_moe_linear.py | 10 +- python/sglang/srt/models/commandr.py | 4 +- python/sglang/srt/models/dbrx.py | 4 +- python/sglang/srt/models/gpt_oss.py | 68 ++-- python/sglang/srt/models/inkling.py | 72 ++-- python/sglang/srt/models/kimi_k3.py | 2 +- python/sglang/srt/models/minimax_m3.py | 10 +- .../test_fp8_moe_weight_gating.py | 29 +- .../quantization/test_int8_linear_methods.py | 5 +- .../layers/test_auxiliary_loader_layouts.py | 162 +++++++++ .../models/test_dbrx_expert_loader_layout.py | 116 ++++++ .../test_gpt_oss_expert_loader_layout.py | 344 ++++++++++++++++++ .../models/test_inkling_moe_loader_layout.py | 263 +++++++++++++ .../models/test_inkling_per_expert_sync.py | 6 +- 17 files changed, 1027 insertions(+), 121 deletions(-) create mode 100644 test/registered/unit/layers/test_auxiliary_loader_layouts.py create mode 100644 test/registered/unit/models/test_dbrx_expert_loader_layout.py create mode 100644 test/registered/unit/models/test_gpt_oss_expert_loader_layout.py create mode 100644 test/registered/unit/models/test_inkling_moe_loader_layout.py diff --git a/python/sglang/srt/layers/activation.py b/python/sglang/srt/layers/activation.py index 375dcde960..d1f277319c 100644 --- a/python/sglang/srt/layers/activation.py +++ b/python/sglang/srt/layers/activation.py @@ -431,6 +431,7 @@ class ScaledActivation(nn.Module): self.act = act_module self.input_is_parallel = input_is_parallel if input_is_parallel: + self.tp_rank = get_parallel().tp_rank tp_size = get_parallel().tp_size intermediate_size_per_partition = divide(intermediate_size, tp_size) else: @@ -448,9 +449,8 @@ class ScaledActivation(nn.Module): def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data if self.input_is_parallel: - tp_rank = get_parallel().tp_rank shard_size = param_data.shape[0] - start_idx = tp_rank * shard_size + start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(0, start_idx, shard_size) assert param_data.shape == loaded_weight.shape param_data.copy_(loaded_weight) diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8_moe.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8_moe.py index 1e0ac87889..9f8817df0d 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8_moe.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8_moe.py @@ -26,7 +26,6 @@ from sglang.srt.layers.quantization.utils import ( per_tensor_dequantize, swap_w13_to_w31, ) -from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import get_bool_env_var, is_hip, set_weight_attrs if TYPE_CHECKING: @@ -98,7 +97,7 @@ class CompressedTensorsW8A8Fp8MoE(CompressedTensorsMoEScheme): if self.block_quant: assert self.weight_block_size is not None layer.weight_block_size = self.weight_block_size - tp_size = get_parallel().tp_size + tp_size = layer.moe_tp_size block_n, block_k = ( self.weight_block_size[0], self.weight_block_size[1], diff --git a/python/sglang/srt/layers/quantization/quark/weights.py b/python/sglang/srt/layers/quantization/quark/weights.py index beaef8b896..619077cda8 100644 --- a/python/sglang/srt/layers/quantization/quark/weights.py +++ b/python/sglang/srt/layers/quantization/quark/weights.py @@ -5,7 +5,6 @@ import re import torch -from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import is_cuda _is_cuda = is_cuda() @@ -61,33 +60,12 @@ def _load_gptoss_quark_expert_weights(model, weights, quark_expert_pat): loaded_params: set[str] = set() mxfp4_block = 32 - moe_tp_rank = get_parallel().moe_tp_rank - moe_tp_size = get_parallel().moe_tp_size - moe_ep_rank = get_parallel().moe_ep_rank - moe_ep_size = get_parallel().moe_ep_size - intermediate_size = model.config.intermediate_size assert intermediate_size % mxfp4_block == 0, ( f"{intermediate_size=} must be divisible by {mxfp4_block=}" ) intermediate_size_block = intermediate_size // mxfp4_block - per_rank_intermediate_size_block = math.ceil(intermediate_size_block / moe_tp_size) - - per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block - - # Calculate common slicing bounds for current rank - assert model.config.num_local_experts % moe_ep_size == 0 - moe_num_local_experts = model.config.num_local_experts // moe_ep_size - - moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size - moe_tp_rank_end = min( - (moe_tp_rank + 1) * per_rank_intermediate_size, intermediate_size - ) - - moe_ep_rank_start = moe_ep_rank * moe_num_local_experts - moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts - for name, weight in weights: # Quark stores experts separately as # `experts.{N}.{gate_up_proj|down_proj}.{suffix}`; pull the @@ -96,6 +74,30 @@ def _load_gptoss_quark_expert_weights(model, weights, quark_expert_pat): if m is None: continue prefix, expert_str, proj, suffix = m.groups() + experts = model.get_submodule(prefix) + moe_tp_rank = experts.moe_tp_rank + moe_tp_size = experts.moe_tp_size + moe_ep_rank = experts.moe_ep_rank + moe_ep_size = experts.moe_ep_size + + per_rank_intermediate_size_block = math.ceil( + intermediate_size_block / moe_tp_size + ) + + per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block + + # Calculate common slicing bounds for current rank + assert model.config.num_local_experts % moe_ep_size == 0 + moe_num_local_experts = model.config.num_local_experts // moe_ep_size + + moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size + moe_tp_rank_end = min( + (moe_tp_rank + 1) * per_rank_intermediate_size, intermediate_size + ) + + moe_ep_rank_start = moe_ep_rank * moe_num_local_experts + moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts + global_expert_id = int(expert_str) if global_expert_id < moe_ep_rank_start or global_expert_id >= moe_ep_rank_end: continue diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 4e238bfa72..f297615166 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -386,17 +386,17 @@ class BailingGroupRMSNormGate(RMSNormGated): dtype=dtype, activation="sigmoid", ) + self.tp_size = get_parallel().attn_tp_size + self.tp_rank = get_parallel().attn_tp_rank self.weight.weight_loader = self.weight_loader - @staticmethod def weight_loader( + self, param: torch.nn.Parameter, loaded_weight: torch.Tensor, ) -> None: - tp_size = get_parallel().attn_tp_size - tp_rank = get_parallel().attn_tp_rank - shard_size = loaded_weight.shape[0] // tp_size - shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size) + shard_size = loaded_weight.shape[0] // self.tp_size + shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size) param.data.copy_(loaded_weight[shard].contiguous()) return diff --git a/python/sglang/srt/models/commandr.py b/python/sglang/srt/models/commandr.py index 4b899c6321..596fe6dc11 100644 --- a/python/sglang/srt/models/commandr.py +++ b/python/sglang/srt/models/commandr.py @@ -84,6 +84,7 @@ class LayerNorm(nn.Module): def __init__(self, param_shape=None, eps=1e-5): super().__init__() self.weight = nn.Parameter(torch.ones(param_shape)) + self.tp_rank = get_parallel().tp_rank if self.weight.dim() != 1 else 0 self.variance_epsilon = eps set_weight_attrs(self.weight, {"weight_loader": self.weight_loader}) @@ -94,12 +95,11 @@ class LayerNorm(nn.Module): return hidden_states, residuals def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): - tp_rank = get_parallel().tp_rank shard_dim = 0 if param.dim() != 1 else None param_data = param.data if shard_dim is not None: shard_size = param_data.shape[shard_dim] - start_idx = tp_rank * shard_size + start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(shard_dim, start_idx, shard_size) assert param_data.shape == loaded_weight.shape param_data.copy_(loaded_weight) diff --git a/python/sglang/srt/models/dbrx.py b/python/sglang/srt/models/dbrx.py index 8bfbf52de2..bc8c7ef8bd 100644 --- a/python/sglang/srt/models/dbrx.py +++ b/python/sglang/srt/models/dbrx.py @@ -103,6 +103,7 @@ class DbrxExperts(nn.Module): ): super().__init__() self.tp_size = get_parallel().tp_size + self.tp_rank = get_parallel().tp_rank self.num_total_experts = config.ffn_config.moe_num_experts self.top_k = config.ffn_config.moe_top_k self.d_model = config.d_model @@ -155,10 +156,9 @@ class DbrxExperts(nn.Module): def weight_loader( self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str ): - tp_rank = get_parallel().tp_rank param_data = param.data shard_size = self.intermediate_size - shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size) + shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size) # DBRX uses GLU for each experts. # GLU has 3 linear layers: w1, v1 and w2. if weight_name.endswith("w1"): diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 0befe23721..44228d2412 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -942,11 +942,6 @@ class GptOssForCausalLM(nn.Module): loaded_params: set[str] = set() mxfp4_block = 32 - moe_tp_rank = get_parallel().moe_tp_rank - moe_tp_size = get_parallel().moe_tp_size - moe_ep_rank = get_parallel().moe_ep_rank - moe_ep_size = get_parallel().moe_ep_size - intermediate_size = self.config.intermediate_size original_intermediate_size = getattr( self.config, "original_intermediate_size", intermediate_size @@ -956,30 +951,49 @@ class GptOssForCausalLM(nn.Module): ) intermediate_size_block = intermediate_size // mxfp4_block - per_rank_intermediate_size_block = math.ceil( - intermediate_size_block / moe_tp_size - ) - - per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block - - # Calculate common slicing bounds for current rank - assert self.config.num_local_experts % moe_ep_size == 0 - moe_num_global_experts = self.config.num_local_experts - moe_num_local_experts = self.config.num_local_experts // moe_ep_size - - moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size - moe_tp_rank_end = min( - (moe_tp_rank + 1) * per_rank_intermediate_size, original_intermediate_size - ) - - moe_ep_rank_start = moe_ep_rank * moe_num_local_experts - moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts - weight_device = next(iter(params_dict.values())).device for name, weight in weights: weight = weight.to(weight_device) + if not any( + suffix in name + for suffix in ( + "gate_up_proj_blocks", + "down_proj_blocks", + "gate_up_proj_scales", + "down_proj_scales", + "gate_up_proj_bias", + "down_proj_bias", + ) + ): + continue + experts = self.get_submodule(name.rsplit(".", 1)[0]) + moe_tp_rank = experts.moe_tp_rank + moe_tp_size = experts.moe_tp_size + moe_ep_rank = experts.moe_ep_rank + moe_ep_size = experts.moe_ep_size + + per_rank_intermediate_size_block = math.ceil( + intermediate_size_block / moe_tp_size + ) + + per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block + + # Calculate common slicing bounds for current rank + assert self.config.num_local_experts % moe_ep_size == 0 + moe_num_global_experts = self.config.num_local_experts + moe_num_local_experts = self.config.num_local_experts // moe_ep_size + + moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size + moe_tp_rank_end = min( + (moe_tp_rank + 1) * per_rank_intermediate_size, + original_intermediate_size, + ) + + moe_ep_rank_start = moe_ep_rank * moe_num_local_experts + moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts + if "gate_up_proj_blocks" in name: # Handle MLP gate and up projection weights new_name = name.replace("gate_up_proj_blocks", "w13_weight") @@ -1240,8 +1254,10 @@ class GptOssForCausalLM(nn.Module): weight_loader = param.weight_loader if "bias" not in name: loaded_weight = loaded_weight.transpose(-2, -1) - if "w2_weight_bias" in name and get_parallel().moe_tp_rank != 0: - loaded_weight = loaded_weight.zero_() + if "w2_weight_bias" in name: + experts = self.get_submodule(name.rsplit(".", 1)[0]) + if experts.moe_tp_rank != 0: + loaded_weight = loaded_weight.zero_() weight_loader( param, diff --git a/python/sglang/srt/models/inkling.py b/python/sglang/srt/models/inkling.py index 92ab78a641..63944d6197 100644 --- a/python/sglang/srt/models/inkling.py +++ b/python/sglang/srt/models/inkling.py @@ -100,29 +100,20 @@ STACKED_DENSE_PARAMS_MAPPING = [ ("down_proj", "down_proj", None), ] -# Online RL weight-sync streams routed experts one at a time as FULL (unsharded) -# per-expert tensors named `...mlp.experts.{j}.gate_proj/up_proj/down_proj.weight`. -# Disk checkpoints only ever carry the fused w13_weight/w2_weight, so this pattern -# never fires on the ordinary loading path. +# Online RL updates use full per-expert gate/up/down tensors; disk checkpoints +# contain fused w13/w2 weights and do not match this pattern. _PER_EXPERT_WEIGHT_RE = re.compile( r"^(?P.+\.mlp\.experts)\.(?P\d+)\.(?Pgate_proj|up_proj|down_proj)\.weight$" ) def _shard_full_to_local( - loaded_weight: torch.Tensor, dst: torch.Tensor, dim: int + loaded_weight: torch.Tensor, dst: torch.Tensor, dim: int, *, tp_rank: int ) -> torch.Tensor: - """Slice a FULL (unsharded) per-expert weight to this MoE-TP rank's shard along `dim`. - - The online weight-sync ships full per-expert tensors (parallelism-agnostic HF - layout); sglang owns its own MoE-TP sharding, so narrow here per - get_parallel().moe_tp_rank. With TP1 the dims already match and this is the - identity, so the ordinary path is byte-for-byte unchanged. - """ + """Shard full HF per-expert updates with the owning module's MoE TP rank.""" if loaded_weight.shape[dim] == dst.shape[dim]: return loaded_weight - rank = get_parallel().moe_tp_rank - return loaded_weight.narrow(dim, rank * dst.shape[dim], dst.shape[dim]) + return loaded_weight.narrow(dim, tp_rank * dst.shape[dim], dst.shape[dim]) KV_REPLICATED_SUFFIXES = ( @@ -1238,12 +1229,10 @@ class InklingForConditionalGeneration(nn.Module): return False param = params_dict[name] if loaded_weight.shape != param.shape: - # shared experts shard over the full tp group; routed over moe_tp - tp_rank = ( - get_parallel().tp_rank - if ".shared_experts" in name - else get_parallel().moe_tp_rank - ) + # Both routed and shared modules own their constructed TP layout. + module = self.get_submodule(name.rsplit(".", 1)[0]) + module = getattr(module, "base_layer", module) + tp_rank = module.moe_tp_rank for dim in range(loaded_weight.ndim): if loaded_weight.shape[dim] == param.shape[dim]: continue @@ -1297,10 +1286,8 @@ class InklingForConditionalGeneration(nn.Module): hold only its contiguous slice (the fused loader shards the intermediate dim only). No-op when EP is off or for replicated shared-expert tensors. """ - ep_size = get_parallel().moe_ep_size if ( - ep_size <= 1 - or ".experts." not in name + ".experts." not in name # per-expert RL sync tensors do their own EP remap in _load_per_expert_param; # a full per-expert tensor whose dim 0 happens to equal n_routed_experts must # not be pre-narrowed here. @@ -1309,8 +1296,17 @@ class InklingForConditionalGeneration(nn.Module): or loaded_weight.shape[0] != self.text_config.n_routed_experts ): return loaded_weight - local = self.text_config.n_routed_experts // ep_size - start = get_parallel().moe_ep_rank * local + prefix = name.split(".experts.", 1)[0] + ".experts" + try: + module = self.get_submodule(prefix) + except AttributeError: + # Checkpoints may contain a layer absent from this model. + return loaded_weight + module = getattr(module, "base_layer", module) + if module.moe_ep_size <= 1: + return loaded_weight + local = self.text_config.n_routed_experts // module.moe_ep_size + start = module.moe_ep_rank * local return loaded_weight.narrow(0, start, local).contiguous() def _load_fused_moe_param( @@ -1403,10 +1399,10 @@ class InklingForConditionalGeneration(nn.Module): f"per-expert RL weight-sync does not support the trtllm MoE layout ({target}); " "serve RL rollouts with the triton MoE runner" ) - ep_size = get_parallel().moe_ep_size + ep_size = moe.moe_ep_size if ep_size > 1: local = self.text_config.n_routed_experts // ep_size - first = get_parallel().moe_ep_rank * local + first = moe.moe_ep_rank * local if not (first <= eid < first + local): loaded_params.add(target) # another rank owns this expert return True @@ -1415,7 +1411,9 @@ class InklingForConditionalGeneration(nn.Module): dst = params_dict[target].data[ eid ] # [H, I_local]; shard intermediate (dim 1) - dst.copy_(_shard_full_to_local(loaded_weight, dst, dim=1)) + dst.copy_( + _shard_full_to_local(loaded_weight, dst, dim=1, tp_rank=moe.moe_tp_rank) + ) else: w13 = params_dict[target].data[ eid @@ -1429,7 +1427,9 @@ class InklingForConditionalGeneration(nn.Module): dst = w13[idx * half : (idx + 1) * half] # contiguous [gate || up] else: dst = w13[idx::2] # Inkling-interleaved rows - dst.copy_(_shard_full_to_local(loaded_weight, dst, dim=0)) + dst.copy_( + _shard_full_to_local(loaded_weight, dst, dim=0, tp_rank=moe.moe_tp_rank) + ) loaded_params.add(target) return True @@ -1590,18 +1590,21 @@ class InklingForConditionalGeneration(nn.Module): continue if ".experts.w13_weight" in name: + experts = modules_dict.get(name.rsplit(".", 1)[0]) + experts = getattr(experts, "base_layer", experts) # bf16 routed layers run the stock FusedMoE forward (not moe_tp_forward) # under --enable-lora, or natively on trtllm_routed for UNQUANTIZED # checkpoints: de-interleave per moe_tp block for the stock weight prep. if ( - loaded_weight.dtype != torch.uint8 + experts is not None + and loaded_weight.dtype != torch.uint8 and self.text_config.inference_moe_w13_interleaved and ( lora_compatible_layout_enabled() or bf16_routed_uses_stock_fused_moe(self.quant_config) ) ): - tp = get_parallel().moe_tp_size + tp = experts.moe_tp_size n_e, two_f, hid = loaded_weight.shape loaded_weight = deinterleave_gate_up( loaded_weight.view(n_e, tp, two_f // tp, hid), dim=2 @@ -1625,16 +1628,19 @@ class InklingForConditionalGeneration(nn.Module): ): continue if ".shared_experts.shared_w13_weight" in name: + experts = modules_dict.get(name.rsplit(".", 1)[0]) + experts = getattr(experts, "base_layer", experts) # InklingSharedFusedMoE's bf16 path needs contiguous [gate||up] w13 for the # SRT runner's silu_and_mul, unlike the interleaved bmm/moe_tp_forward paths. # Per-rank blocks are sized by the FULL tp group (InklingSharedFusedMoE always # shards over it at EP=1), NOT moe_tp (= tp/ep, wrong under --ep-size > 1). if ( - loaded_weight.dtype != torch.uint8 + experts is not None + and loaded_weight.dtype != torch.uint8 and self.text_config.inference_moe_w13_interleaved and use_inkling_shared_fused_moe() ): - tp = get_parallel().tp_size + tp = experts.moe_tp_size n_e, two_f, hid = loaded_weight.shape loaded_weight = deinterleave_gate_up( loaded_weight.view(n_e, tp, two_f // tp, hid), dim=2 diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 689004c50a..99fb96f783 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -1739,7 +1739,7 @@ class KimiK3DeltaAttention(nn.Module): def _a_log_weight_loader( param: torch.Tensor, loaded_weight: torch.Tensor ) -> None: - tp_rank = get_parallel().attn_tp_rank + tp_rank = self.attn_tp_rank shard_size = param.data.shape[2] # local_num_heads start_idx = tp_rank * shard_size diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 7a1b4801d5..c9109ed330 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -173,6 +173,7 @@ class MultiHeadRMSNorm(nn.Module): ) -> None: super().__init__() self.tp_world = get_parallel().attn_tp_size + self.tp_rank = get_parallel().attn_tp_rank self.num_heads = num_heads self.num_heads_per_tp = num_heads // self.tp_world self.head_dim = head_dim @@ -183,16 +184,13 @@ class MultiHeadRMSNorm(nn.Module): self.apply_layernorm_1p = apply_layernorm_1p self.variance_epsilon = eps - @staticmethod def weight_loader( + self, param: nn.Parameter, loaded_weight: torch.Tensor, ) -> None: - tp_world = get_parallel().attn_tp_size - tp_rank = get_parallel().attn_tp_rank - - shard_size = loaded_weight.shape[0] // tp_world - shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size) + shard_size = loaded_weight.shape[0] // self.tp_world + shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size) param.data.copy_(loaded_weight[shard].reshape_as(param)) def forward( diff --git a/test/registered/unit/layers/quantization/test_fp8_moe_weight_gating.py b/test/registered/unit/layers/quantization/test_fp8_moe_weight_gating.py index 85f4bb1ced..09428cc325 100644 --- a/test/registered/unit/layers/quantization/test_fp8_moe_weight_gating.py +++ b/test/registered/unit/layers/quantization/test_fp8_moe_weight_gating.py @@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=10, suite="base-a-test-cpu") import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import torch @@ -31,6 +31,7 @@ class _RecordingLayer: def __init__(self, is_gated: bool): self.moe_runner_config = MoeRunnerConfig(is_gated=is_gated) + self.moe_tp_size = 1 self.params = {} def register_parameter(self, name, param): @@ -47,20 +48,18 @@ def _create_weights(is_gated: bool, block_quant: bool): is_checkpoint_fp8_serialized=False, ) - with patch.object(fp8_quant, "get_parallel") as parallel: - parallel.return_value.tp_size = 1 - fp8_quant.Fp8MoEMethod.create_fp8_moe_weight_( - layer=layer, - num_experts=NUM_EXPERTS, - hidden_size=HIDDEN, - intermediate_size_per_partition=INTERMEDIATE, - block_quant=block_quant, - quant_config=quant_config, - use_mxfp8=False, - is_checkpoint_fp8_serialized=False, - is_fp4_expert=False, - params_dtype=torch.bfloat16, - ) + fp8_quant.Fp8MoEMethod.create_fp8_moe_weight_( + layer=layer, + num_experts=NUM_EXPERTS, + hidden_size=HIDDEN, + intermediate_size_per_partition=INTERMEDIATE, + block_quant=block_quant, + quant_config=quant_config, + use_mxfp8=False, + is_checkpoint_fp8_serialized=False, + is_fp4_expert=False, + params_dtype=torch.bfloat16, + ) return layer.params diff --git a/test/registered/unit/layers/quantization/test_int8_linear_methods.py b/test/registered/unit/layers/quantization/test_int8_linear_methods.py index fa6f04329f..cd00e6f254 100644 --- a/test/registered/unit/layers/quantization/test_int8_linear_methods.py +++ b/test/registered/unit/layers/quantization/test_int8_linear_methods.py @@ -12,7 +12,6 @@ import torch from sglang.srt.layers.quantization.blockwise_int8 import BlockInt8Config from sglang.srt.layers.quantization.w8a8_int8 import W8A8Int8Config -from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import get_device_sm from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.layer_ut_utils import ( @@ -113,9 +112,7 @@ class TestBlockInt8Linear(_Int8LinearCheck): activation_scheme="dynamic", weight_block_size=[128, 128], ) - # create_weights reads get_parallel().tp_size, not the layer's argument. - with get_parallel().override(tp_size=1, tp_rank=0): - layer = make_tp1_column_parallel_linear(quant_config, n, k) + layer = make_tp1_column_parallel_linear(quant_config, n, k) w = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) / 10 w_int8, scale_inv, w_dequant = _quantize_int8_block(w) load_linear_weights(layer, weight=w_int8, weight_scale_inv=scale_inv) diff --git a/test/registered/unit/layers/test_auxiliary_loader_layouts.py b/test/registered/unit/layers/test_auxiliary_loader_layouts.py new file mode 100644 index 0000000000..36ef336fb5 --- /dev/null +++ b/test/registered/unit/layers/test_auxiliary_loader_layouts.py @@ -0,0 +1,162 @@ +"""Auxiliary checkpoint shards stay with the constructed activation/norm layer.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace + +import torch +from torch import nn + +from sglang.srt.layers.activation import ScaledActivation +from sglang.srt.models.bailing_moe_linear import BailingGroupRMSNormGate +from sglang.srt.models.commandr import LayerNorm +from sglang.srt.models.kimi_k3 import KimiK3DeltaAttention +from sglang.srt.models.minimax_m3 import MultiHeadRMSNorm +from sglang.srt.runtime_context import SpawnRanks, get_parallel, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.parallel_groups import parallel_scope, publish +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large") + +KINDS = ( + "scaled", + "scaled_replicated", + "command", + "command_replicated", + "bailing", + "minimax", + "minimax_1p", + "kimi", +) + + +def loading_scope(changed): + if not changed: + return nullcontext() + return parallel_scope( + tp_size=1, + tp_rank=0, + tp_group=None, + attn_tp_size=1, + attn_tp_rank=0, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + moe_ep_group=None, + moe_dp_size=1, + moe_dp_rank=0, + ) + + +def build_layer(kind): + parallel = get_parallel() + if kind.startswith("scaled"): + replicated = kind.endswith("replicated") + module = ScaledActivation(nn.GELU(), 32, input_is_parallel=not replicated) + return ( + module, + module.scales, + (32,), + 0 if replicated else parallel.tp_rank, + 1 if replicated else parallel.tp_size, + ) + if kind.startswith("command"): + replicated = kind.endswith("replicated") + module = LayerNorm(16 if replicated else (8 // parallel.tp_size, 16)) + return ( + module, + module.weight, + (16,) if replicated else (8, 16), + 0 if replicated else parallel.tp_rank, + 1 if replicated else parallel.tp_size, + ) + if kind == "bailing": + module = BailingGroupRMSNormGate(64 // parallel.attn_tp_size, group_size=8) + return ( + module, + module.weight, + (64,), + parallel.attn_tp_rank, + parallel.attn_tp_size, + ) + if kind.startswith("minimax"): + module = MultiHeadRMSNorm(8, 16, apply_layernorm_1p=kind.endswith("1p")) + return ( + module, + module.weight, + (8, 16), + parallel.attn_tp_rank, + parallel.attn_tp_size, + ) + config = SimpleNamespace( + dtype=torch.get_default_dtype(), + v_head_dim=8, + rms_norm_eps=1e-6, + linear_attn_config=dict(head_dim=8, num_heads=8, short_conv_kernel_size=3), + ) + module = KimiK3DeltaAttention(0, 32, config) + return module, module.A_log, (8,), parallel.attn_tp_rank, parallel.attn_tp_size + + +def values(shape, param, offset=0): + count = 1 + for size in shape: + count *= size + return ((torch.arange(count, device=param.device) + offset) % 17 + 1).reshape( + shape + ).to(param.dtype) / 32 + + +def load_layer(kind, *, changed=False, flat=False, old_kimi=False, offset=0): + module, param, shape, rank, size = build_layer(kind) + full = values(shape, param, offset) + checkpoint = full.flatten() if flat else full + if old_kimi: + checkpoint = full.view(1, 1, 8, 1) + expected = full.chunk(size, dim=0)[rank].reshape_as(param) + with loading_scope(changed): + param.weight_loader(param, checkpoint) + torch.testing.assert_close(param, expected, rtol=0, atol=0) + return module, param, expected + + +class TestAuxiliaryLoaderLayouts(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def check_loads(self, changed): + for dp in (1, 2): + for rank in range(4): + reset_context() + publish( + ServerArgs( + model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + for kind in KINDS: + for offset in (0, 7): + load_layer(kind, changed=changed, offset=offset) + if kind.startswith("minimax"): + load_layer(kind, changed=changed, flat=True, offset=offset) + if kind == "kimi": + load_layer( + kind, changed=changed, old_kimi=True, offset=offset + ) + + def test_loaders_after_scope_exit(self): + self.check_loads(True) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_dbrx_expert_loader_layout.py b/test/registered/unit/models/test_dbrx_expert_loader_layout.py new file mode 100644 index 0000000000..1e4ce8ea55 --- /dev/null +++ b/test/registered/unit/models/test_dbrx_expert_loader_layout.py @@ -0,0 +1,116 @@ +"""DBRX expert checkpoint shards follow the constructed TP layout.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace + +import torch + +from sglang.srt.layers.linear import ReplicatedLinear +from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod +from sglang.srt.models.dbrx import DbrxExperts +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=15, stage="nightly", runner_config="1-gpu-large") + + +def loading_scope(changed): + if not changed: + return nullcontext() + return get_parallel().override( + tp_size=1, + tp_rank=0, + tp_group=None, + attn_tp_size=1, + attn_tp_rank=0, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + moe_ep_group=None, + moe_dp_size=1, + moe_dp_rank=0, + ) + + +def build_experts(dtype=torch.bfloat16): + config = SimpleNamespace( + d_model=32, + ffn_config=SimpleNamespace(moe_num_experts=4, moe_top_k=2, ffn_hidden_size=128), + ) + module = DbrxExperts(config, 0, params_dtype=dtype) + assert isinstance(module.router.layer, ReplicatedLinear) + assert isinstance(module.router.layer.quant_method, UnquantizedLinearMethod) + return module + + +def values(shape, dtype, offset=0): + count = 1 + for size in shape: + count *= size + return ( + (((torch.arange(count, device="cuda") + offset) % 31 - 15) / 128) + .to(dtype) + .reshape(shape) + ) + + +def load_experts(module, *, changed=False, flat=False, offset=0): + rank = get_parallel().tp_rank + start = rank * module.intermediate_size + stop = start + module.intermediate_size + loaded = {} + for index, name in enumerate(("w1", "v1", "w2")): + full = values((4, 128, 32), module.ws.dtype, offset + index * 7) + param = module.w2s if name == "w2" else module.ws + with loading_scope(changed): + param.weight_loader(param, full.flatten() if flat else full, name) + loaded[name] = full[:, start:stop, :] + expected_ws = torch.cat((loaded["w1"], loaded["v1"]), dim=1) + expected_w2 = loaded["w2"].transpose(1, 2) + torch.testing.assert_close(module.ws, expected_ws, rtol=0, atol=0) + torch.testing.assert_close(module.w2s, expected_w2, rtol=0, atol=0) + return expected_ws, expected_w2 + + +@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU") +class TestDbrxExpertLoaderLayout(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def check_loads(self, changed): + with torch.inference_mode(): + for size in (1, 2, 4): + for rank in range(size): + reset_context() + publish( + ServerArgs(model_path="dummy", device="cuda", tp_size=size), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + for dtype in (torch.float32, torch.bfloat16): + module = build_experts(dtype) + for flat in (False, True): + for offset in (0, 11): + load_experts( + module, + changed=changed, + flat=flat, + offset=offset, + ) + + def test_native_loader_after_scope_exit(self): + self.check_loads(True) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_gpt_oss_expert_loader_layout.py b/test/registered/unit/models/test_gpt_oss_expert_loader_layout.py new file mode 100644 index 0000000000..5c316d5596 --- /dev/null +++ b/test/registered/unit/models/test_gpt_oss_expert_loader_layout.py @@ -0,0 +1,344 @@ +"""GPT-OSS packed expert checkpoints follow the destination expert layout.""" + +import re +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.layers.moe.fused_moe_triton import FusedMoE +from sglang.srt.layers.quantization.mxfp4 import Mxfp4Config, Mxfp4MoEMethod +from sglang.srt.layers.quantization.quark.quark import QuarkConfig +from sglang.srt.layers.quantization.quark.schemes.quark_w4a8_mxfp4_moe import ( + QuarkW4A8MXFp4MoE, +) +from sglang.srt.layers.quantization.quark.weights import ( + _load_gptoss_quark_expert_weights, +) +from sglang.srt.models.gpt_oss import GptOssForCausalLM +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large") + +EXPERT_PATTERN = re.compile( + r"^(.*\.mlp\.experts)\.(\d+)\.(gate_up_proj|down_proj)\." + r"(weight|weight_scale|input_scale|bias)$" +) + + +def loading_scope(changed): + if not changed: + return nullcontext() + return get_parallel().override( + tp_size=1, + tp_rank=0, + tp_group=None, + attn_tp_size=1, + attn_tp_rank=0, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + moe_ep_group=None, + moe_dp_size=1, + moe_dp_rank=0, + ) + + +def quark_config(): + return QuarkConfig( + quant_config={ + "packed_modules_mapping": {}, + "layer_quant_config": {}, + "layer_type_quant_config": {}, + "global_quant_config": { + "weight": { + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "is_dynamic": False, + "scale_format": "e8m0", + }, + "input_tensors": { + "dtype": "fp8_e4m3", + "qscheme": "per_tensor", + "is_dynamic": False, + }, + "output_tensors": None, + "bias": None, + }, + }, + is_prequantized=True, + ) + + +def build_model(kind, intermediate=384, original=None, device="cuda"): + config = SimpleNamespace( + hidden_size=128, + intermediate_size=intermediate, + num_local_experts=4, + num_hidden_layers=1, + ) + if original is not None: + config.original_intermediate_size = original + model = GptOssForCausalLM.__new__(GptOssForCausalLM) + torch.nn.Module.__init__(model) + model.config = config + model.quant_config = ( + Mxfp4Config(is_checkpoint_mxfp4_serialized=True) + if kind == "mxfp4" + else quark_config() + if kind == "quark" + else None + ) + # Quark's W4A8 runner is AMD-only. Keep native parameter construction and + # the checkpoint loader while avoiding runner creation in this CUDA test. + runner_scope = ( + patch.object(QuarkW4A8MXFp4MoE, "create_moe_runner", return_value=None) + if kind == "quark" + else nullcontext() + ) + with torch.device(device), runner_scope: + experts = FusedMoE( + num_experts=4, + top_k=2, + hidden_size=128, + intermediate_size=intermediate, + layer_id=0, + gemm1_alpha=1.702, + gemm1_clamp_limit=10.0, + quant_config=model.quant_config, + params_dtype=torch.bfloat16, + with_bias=True, + use_weight_loader_fused=kind == "unquantized", + reduce_results=False, + prefix="model.layers.0.mlp.experts", + ) + if kind == "mxfp4": + assert isinstance(experts.quant_method, Mxfp4MoEMethod) + elif kind == "quark": + assert isinstance(experts.scheme, QuarkW4A8MXFp4MoE) + model.model = torch.nn.Module() + model.model.layers = torch.nn.ModuleList([torch.nn.Module()]) + model.model.layers[0].mlp = torch.nn.Module() + model.model.layers[0].mlp.experts = experts + return model, experts + + +def values(shape, dtype, offset=0, device="cuda"): + count = 1 + for size in shape: + count *= size + indices = torch.arange(count, device=device) + offset + data = ( + (indices % 16 + (indices // 16 % 16) * 16) + if dtype == torch.uint8 + else (indices % 31 - 15) / 64 + ) + return data.to(dtype).reshape(shape) + + +def load_experts(model, kind, *, changed=False, offset=0): + module = model.model.layers[0].mlp.experts + config = model.config + intermediate = getattr( + config, "original_intermediate_size", config.intermediate_size + ) + hidden = config.hidden_size + device = module.w13_weight.device + source = { + "w13_weight": values( + (4, 2 * intermediate, hidden // 2), torch.uint8, offset, device + ), + "w2_weight": values( + (4, hidden, intermediate // 2), torch.uint8, offset + 3, device + ), + "w13_weight_scale": values( + (4, 2 * intermediate, hidden // 32), torch.uint8, offset + 5, device + ), + "w2_weight_scale": values( + (4, hidden, intermediate // 32), torch.uint8, offset + 7, device + ), + "w13_weight_bias": values( + (4, 2 * intermediate), module.w13_weight_bias.dtype, offset + 11, device + ), + "w2_weight_bias": values( + (4, hidden), module.w2_weight_bias.dtype, offset + 13, device + ), + } + if kind == "quark": + source["w13_input_scale"] = values((4,), torch.float32, offset + 17, device) + source["w2_input_scale"] = values((4,), torch.float32, offset + 19, device) + prefix = "model.layers.0.mlp.experts" + suffixes = ( + ("w13_weight", "gate_up_proj", "blocks"), + ("w2_weight", "down_proj", "blocks"), + ("w13_weight_scale", "gate_up_proj", "scales"), + ("w2_weight_scale", "down_proj", "scales"), + ("w13_weight_bias", "gate_up_proj", "bias"), + ("w2_weight_bias", "down_proj", "bias"), + ) + checkpoint = [] + if kind == "mxfp4": + for param_name, projection, suffix in suffixes: + data = source[param_name] + if suffix == "blocks": + data = data.unflatten(-1, (-1, 16)) + checkpoint.append((f"{prefix}.{projection}_{suffix}", data)) + checkpoint.append((f"{prefix}.unused_tensor", torch.ones(1, device=device))) + else: + for expert in range(4): + for param_name, projection, suffix in suffixes: + suffix = {"blocks": "weight", "scales": "weight_scale"}.get( + suffix, suffix + ) + checkpoint.append( + ( + f"{prefix}.{expert}.{projection}.{suffix}", + source[param_name][expert], + ) + ) + for param_name, projection in ( + ("w13_input_scale", "gate_up_proj"), + ("w2_input_scale", "down_proj"), + ): + checkpoint.append( + ( + f"{prefix}.{expert}.{projection}.input_scale", + source[param_name][expert], + ) + ) + checkpoint.append(("not-an-expert-weight", torch.ones(1, device=device))) + before = {name: param.detach().clone() for name, param in module.named_parameters()} + with loading_scope(changed): + loaded = ( + model._load_mxfp4_experts_weights(iter(checkpoint)) + if kind == "mxfp4" + else _load_gptoss_quark_expert_weights( + model, iter(checkpoint), EXPERT_PATTERN + ) + ) + width = ( + (config.intermediate_size // 32 + module.moe_tp_size - 1) // module.moe_tp_size + ) * 32 + start = module.moe_tp_rank * width + stop = min(start + width, intermediate) + expert_start = module.moe_ep_rank * module.num_local_experts + expert_stop = expert_start + module.num_local_experts + expected = {} + for name, original_data in source.items(): + data = original_data[expert_start:expert_stop] + target = before[name] + if name.startswith("w13") and "input_scale" not in name: + if kind == "mxfp4": + data = data[:, 2 * start : 2 * stop] + if data.dim() == 3: + target[:, : data.shape[1], : data.shape[2]] = data + else: + target[:, : data.shape[1]] = data + else: + gate = data[:, 0::2, ...][:, start:stop] + up = data[:, 1::2, ...][:, start:stop] + half = target.shape[1] // 2 + if data.dim() == 3: + target[:, : gate.shape[1], : gate.shape[2]] = gate + target[:, half : half + up.shape[1], : up.shape[2]] = up + else: + target[:, : gate.shape[1]] = gate + target[:, half : half + up.shape[1]] = up + elif name == "w2_weight": + data = data[..., start // 2 : stop // 2] + target[:, : data.shape[1], : data.shape[2]] = data + elif name == "w2_weight_scale": + data = data[..., start // 32 : stop // 32] + target[:, : data.shape[1], : data.shape[2]] = data + elif name == "w2_weight_bias": + target[:, : data.shape[1]] = ( + data if module.moe_tp_rank == 0 else torch.zeros_like(data) + ) + else: + target.copy_(data) + torch.testing.assert_close(getattr(module, name), target, rtol=0, atol=0) + expected[name] = target + assert loaded == {f"{prefix}.{name}" for name in source} + return expected + + +@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU") +class TestGptOssExpertLoaderLayout(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def check_loads(self, changed): + with torch.inference_mode(): + for ep_size in (1, 2, 4): + for rank in range(4): + reset_context() + publish( + ServerArgs( + model_path="dummy", + device="cuda", + tp_size=4, + ep_size=ep_size, + moe_runner_backend="triton", + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + for kind in ("mxfp4", "quark"): + for intermediate, original in ( + ((384, None), (288, None), (384, 288)) + if kind == "mxfp4" + else ((384, None), (512, None)) + ): + model, _ = build_model(kind, intermediate, original) + for offset in (0, 11): + load_experts( + model, kind, changed=changed, offset=offset + ) + + def test_normal_down_projection_bias_uses_the_expert_rank(self): + with torch.inference_mode(): + for rank in range(4): + reset_context() + publish( + ServerArgs( + model_path="dummy", + device="cuda", + tp_size=4, + moe_runner_backend="triton", + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + model, module = build_model("unquantized") + for changed in (False, True): + bias = values((4, 128), module.w2_weight_bias.dtype) + expected = bias.clone() if rank == 0 else torch.zeros_like(bias) + with loading_scope(changed): + model._load_normal_weights( + [("model.layers.0.mlp.experts.down_proj_bias", bias)], + is_nextn=False, + weight_name_mapping=None, + ) + torch.testing.assert_close( + module.w2_weight_bias, expected, rtol=0, atol=0 + ) + + def test_native_loaders_after_scope_exit(self): + self.check_loads(True) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_inkling_moe_loader_layout.py b/test/registered/unit/models/test_inkling_moe_loader_layout.py new file mode 100644 index 0000000000..4c43902d87 --- /dev/null +++ b/test/registered/unit/models/test_inkling_moe_loader_layout.py @@ -0,0 +1,263 @@ +"""Inkling checkpoint preprocessing follows its constructed expert modules.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +from torch import nn + +from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE +from sglang.srt.lora.layers import BaseLayerWithLoRA +from sglang.srt.models import inkling +from sglang.srt.models.inkling_common.moe import InklingSharedFusedMoE +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") + + +def values(shape, offset=0, *, device="cpu", dtype=torch.float32): + count = 1 + for size in shape: + count *= size + return ( + (((torch.arange(count, device=device) + offset) % 29 - 14) / 128) + .reshape(shape) + .to(dtype) + ) + + +def loading_scope(changed): + if not changed: + return nullcontext() + return get_parallel().override( + tp_size=1, + tp_rank=0, + tp_group=None, + attn_tp_size=1, + attn_tp_rank=0, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + moe_ep_group=None, + moe_dp_size=1, + moe_dp_rank=0, + ) + + +def build_model(*, width=32, intermediate=64, wrapped=False): + model = inkling.InklingForConditionalGeneration.__new__( + inkling.InklingForConditionalGeneration + ) + nn.Module.__init__(model) + model.text_config = SimpleNamespace( + n_routed_experts=8, inference_moe_w13_interleaved=True + ) + model.quant_config = model.audio = model.visual = None + model.llm = nn.Module() + block = nn.Module() + block.mlp = nn.Module() + routed = FusedMoE( + num_experts=8, + hidden_size=width, + intermediate_size=intermediate, + layer_id=0, + top_k=1, + use_weight_loader_fused=True, + ) + shared = InklingSharedFusedMoE( + 2, width, intermediate, 0, "llm.layers.0.mlp.shared_experts", None, True + ) + block.mlp.experts = BaseLayerWithLoRA(routed, Mock()) if wrapped else routed + block.mlp.shared_experts = shared + model.llm.layers = nn.ModuleList([block]) + return model, routed, shared + + +def load_per_expert(model, routed, *, changed=False, contiguous=False, offset=0): + width = routed.hidden_size + intermediate = routed.intermediate_size_per_partition * routed.moe_tp_size + local = 8 // routed.moe_ep_size + start = routed.moe_ep_rank * local + shard = routed.moe_tp_rank * routed.intermediate_size_per_partition + weights = [] + gates = [] + ups = [] + downs = [] + for expert in range(8): + gate = values( + (intermediate, width), + offset + 3 * expert, + device=routed.w13_weight.device, + dtype=routed.w13_weight.dtype, + ) + up = values( + (intermediate, width), + offset + 3 * expert + 1, + device=gate.device, + dtype=gate.dtype, + ) + down = values( + (width, intermediate), + offset + 3 * expert + 2, + device=gate.device, + dtype=gate.dtype, + ) + for name, weight in (("gate_proj", gate), ("up_proj", up), ("down_proj", down)): + weights.append( + (f"model.llm.layers.0.mlp.experts.{expert}.{name}.weight", weight) + ) + if start <= expert < start + local: + gates.append(gate[shard : shard + routed.intermediate_size_per_partition]) + ups.append(up[shard : shard + routed.intermediate_size_per_partition]) + downs.append( + down[:, shard : shard + routed.intermediate_size_per_partition] + ) + with ( + loading_scope(changed), + patch.object( + inkling, "lora_compatible_layout_enabled", return_value=contiguous + ), + ): + loaded = model.load_weights(weights) + gate, up = torch.stack(gates), torch.stack(ups) + expected_w13 = ( + torch.cat((gate, up), dim=1) + if contiguous + else torch.stack((gate, up), dim=2).flatten(1, 2) + ) + expected_w2 = torch.stack(downs) + suffix = ( + ".base_layer" + if isinstance(model.llm.layers[0].mlp.experts, BaseLayerWithLoRA) + else "" + ) + assert loaded == { + f"llm.layers.0.mlp.experts{suffix}.w13_weight", + f"llm.layers.0.mlp.experts{suffix}.w2_weight", + } + torch.testing.assert_close(routed.w13_weight, expected_w13, rtol=0, atol=0) + torch.testing.assert_close(routed.w2_weight, expected_w2, rtol=0, atol=0) + return expected_w13, expected_w2 + + +def load_fused(model, routed, shared, *, changed=False, offset=0): + weights = [] + expected = [] + for kind, module, experts in ( + ("experts", routed, 8), + ("shared_experts", shared, 2), + ): + width = module.hidden_size + intermediate = module.intermediate_size_per_partition * module.moe_tp_size + w13 = values( + (experts, 2 * intermediate, width), + offset, + device=module.w13_weight.device, + dtype=module.w13_weight.dtype, + ) + w2 = values( + (experts, width, intermediate), + offset + 1, + device=w13.device, + dtype=w13.dtype, + ) + prefix = f"model.llm.layers.0.mlp.{kind}." + weights.extend( + ( + ( + prefix + + ( + "shared_w13_weight" + if kind == "shared_experts" + else "w13_weight" + ), + w13, + ), + ( + prefix + + ("shared_w2_weight" if kind == "shared_experts" else "w2_weight"), + w2, + ), + ) + ) + local = experts // module.moe_ep_size + first = module.moe_ep_rank * local + rank = module.moe_tp_rank + local_w13 = w13[first : first + local].chunk(module.moe_tp_size, dim=1)[rank] + local_w13 = torch.cat((local_w13[:, 0::2], local_w13[:, 1::2]), dim=1) + local_w2 = w2[first : first + local].chunk(module.moe_tp_size, dim=2)[rank] + expected.append((module, local_w13, local_w2)) + weights.append(("model.llm.layers.999.mlp.experts.w13_weight", weights[0][1])) + with ( + loading_scope(changed), + patch.object(inkling, "lora_compatible_layout_enabled", return_value=True), + patch.object(inkling, "use_inkling_shared_fused_moe", return_value=True), + ): + loaded = model.load_weights(weights) + assert loaded == { + f"llm.layers.0.mlp.{kind}.{leaf}" + for kind in ("experts", "shared_experts") + for leaf in ("w13_weight", "w2_weight") + } + for module, w13, w2 in expected: + torch.testing.assert_close(module.w13_weight, w13, rtol=0, atol=0) + torch.testing.assert_close(module.w2_weight, w2, rtol=0, atol=0) + return expected + + +class TestInklingMoeLoaderLayout(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + + def check_layouts(self, *, changed=False, wrapped=False): + for ep in (1, 2, 4): + for rank in range(4): + publish( + ServerArgs( + model_path="dummy", + device="cpu", + tp_size=4, + ep_size=ep, + moe_runner_backend="triton", + ), + role="test", + ranks=SpawnRanks(world_rank=rank), + ) + for contiguous in (False, True): + model, routed, _ = build_model(wrapped=wrapped) + for offset in (0, 7): + load_per_expert( + model, + routed, + changed=changed, + contiguous=contiguous, + offset=offset, + ) + if not wrapped: + model, routed, shared = build_model() + for offset in (0, 7): + load_fused( + model, routed, shared, changed=changed, offset=offset + ) + + def test_native_expert_loaders_after_scope_exit(self): + self.check_layouts(changed=True) + + def test_native_per_expert_loader_with_lora_parameter_aliases(self): + self.check_layouts(changed=True, wrapped=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_inkling_per_expert_sync.py b/test/registered/unit/models/test_inkling_per_expert_sync.py index 5b1d7b325a..3987a1a7ff 100644 --- a/test/registered/unit/models/test_inkling_per_expert_sync.py +++ b/test/registered/unit/models/test_inkling_per_expert_sync.py @@ -20,7 +20,7 @@ import unittest import torch import sglang.srt.models.inkling as inkling_mod -from sglang.srt.runtime_context import reset_context +from sglang.srt.runtime_context import get_parallel, reset_context from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import publish_build_topology @@ -38,6 +38,10 @@ class _FakeModel: inference_moe_w13_interleaved=interleaved, ) self._moe = moe if moe is not None else types.SimpleNamespace() + self._moe.moe_ep_size = get_parallel().moe_ep_size + self._moe.moe_ep_rank = get_parallel().moe_ep_rank + self._moe.moe_tp_size = get_parallel().moe_tp_size + self._moe.moe_tp_rank = get_parallel().moe_tp_rank def get_submodule(self, path): return self._moe