[Refactor] Load auxiliary and expert checkpoints using owner partitions (#42589)

This commit is contained in:
Cheng Wan
2026-10-07 16:24:03 -07:00
committed by GitHub
parent 64c7f65eac
commit 5e12831cec
17 changed files with 1027 additions and 121 deletions
+2 -2
View File
@@ -431,6 +431,7 @@ class ScaledActivation(nn.Module):
self.act = act_module
self.input_is_parallel = input_is_parallel
if input_is_parallel:
self.tp_rank = get_parallel().tp_rank
tp_size = get_parallel().tp_size
intermediate_size_per_partition = divide(intermediate_size, tp_size)
else:
@@ -448,9 +449,8 @@ class ScaledActivation(nn.Module):
def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor):
param_data = param.data
if self.input_is_parallel:
tp_rank = get_parallel().tp_rank
shard_size = param_data.shape[0]
start_idx = tp_rank * shard_size
start_idx = self.tp_rank * shard_size
loaded_weight = loaded_weight.narrow(0, start_idx, shard_size)
assert param_data.shape == loaded_weight.shape
param_data.copy_(loaded_weight)
@@ -26,7 +26,6 @@ from sglang.srt.layers.quantization.utils import (
per_tensor_dequantize,
swap_w13_to_w31,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_bool_env_var, is_hip, set_weight_attrs
if TYPE_CHECKING:
@@ -98,7 +97,7 @@ class CompressedTensorsW8A8Fp8MoE(CompressedTensorsMoEScheme):
if self.block_quant:
assert self.weight_block_size is not None
layer.weight_block_size = self.weight_block_size
tp_size = get_parallel().tp_size
tp_size = layer.moe_tp_size
block_n, block_k = (
self.weight_block_size[0],
self.weight_block_size[1],
@@ -5,7 +5,6 @@ import re
import torch
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_cuda
_is_cuda = is_cuda()
@@ -61,33 +60,12 @@ def _load_gptoss_quark_expert_weights(model, weights, quark_expert_pat):
loaded_params: set[str] = set()
mxfp4_block = 32
moe_tp_rank = get_parallel().moe_tp_rank
moe_tp_size = get_parallel().moe_tp_size
moe_ep_rank = get_parallel().moe_ep_rank
moe_ep_size = get_parallel().moe_ep_size
intermediate_size = model.config.intermediate_size
assert intermediate_size % mxfp4_block == 0, (
f"{intermediate_size=} must be divisible by {mxfp4_block=}"
)
intermediate_size_block = intermediate_size // mxfp4_block
per_rank_intermediate_size_block = math.ceil(intermediate_size_block / moe_tp_size)
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
# Calculate common slicing bounds for current rank
assert model.config.num_local_experts % moe_ep_size == 0
moe_num_local_experts = model.config.num_local_experts // moe_ep_size
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
moe_tp_rank_end = min(
(moe_tp_rank + 1) * per_rank_intermediate_size, intermediate_size
)
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
for name, weight in weights:
# Quark stores experts separately as
# `experts.{N}.{gate_up_proj|down_proj}.{suffix}`; pull the
@@ -96,6 +74,30 @@ def _load_gptoss_quark_expert_weights(model, weights, quark_expert_pat):
if m is None:
continue
prefix, expert_str, proj, suffix = m.groups()
experts = model.get_submodule(prefix)
moe_tp_rank = experts.moe_tp_rank
moe_tp_size = experts.moe_tp_size
moe_ep_rank = experts.moe_ep_rank
moe_ep_size = experts.moe_ep_size
per_rank_intermediate_size_block = math.ceil(
intermediate_size_block / moe_tp_size
)
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
# Calculate common slicing bounds for current rank
assert model.config.num_local_experts % moe_ep_size == 0
moe_num_local_experts = model.config.num_local_experts // moe_ep_size
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
moe_tp_rank_end = min(
(moe_tp_rank + 1) * per_rank_intermediate_size, intermediate_size
)
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
global_expert_id = int(expert_str)
if global_expert_id < moe_ep_rank_start or global_expert_id >= moe_ep_rank_end:
continue
@@ -386,17 +386,17 @@ class BailingGroupRMSNormGate(RMSNormGated):
dtype=dtype,
activation="sigmoid",
)
self.tp_size = get_parallel().attn_tp_size
self.tp_rank = get_parallel().attn_tp_rank
self.weight.weight_loader = self.weight_loader
@staticmethod
def weight_loader(
self,
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
) -> None:
tp_size = get_parallel().attn_tp_size
tp_rank = get_parallel().attn_tp_rank
shard_size = loaded_weight.shape[0] // tp_size
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
shard_size = loaded_weight.shape[0] // self.tp_size
shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size)
param.data.copy_(loaded_weight[shard].contiguous())
return
+2 -2
View File
@@ -84,6 +84,7 @@ class LayerNorm(nn.Module):
def __init__(self, param_shape=None, eps=1e-5):
super().__init__()
self.weight = nn.Parameter(torch.ones(param_shape))
self.tp_rank = get_parallel().tp_rank if self.weight.dim() != 1 else 0
self.variance_epsilon = eps
set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
@@ -94,12 +95,11 @@ class LayerNorm(nn.Module):
return hidden_states, residuals
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
tp_rank = get_parallel().tp_rank
shard_dim = 0 if param.dim() != 1 else None
param_data = param.data
if shard_dim is not None:
shard_size = param_data.shape[shard_dim]
start_idx = tp_rank * shard_size
start_idx = self.tp_rank * shard_size
loaded_weight = loaded_weight.narrow(shard_dim, start_idx, shard_size)
assert param_data.shape == loaded_weight.shape
param_data.copy_(loaded_weight)
+2 -2
View File
@@ -103,6 +103,7 @@ class DbrxExperts(nn.Module):
):
super().__init__()
self.tp_size = get_parallel().tp_size
self.tp_rank = get_parallel().tp_rank
self.num_total_experts = config.ffn_config.moe_num_experts
self.top_k = config.ffn_config.moe_top_k
self.d_model = config.d_model
@@ -155,10 +156,9 @@ class DbrxExperts(nn.Module):
def weight_loader(
self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str
):
tp_rank = get_parallel().tp_rank
param_data = param.data
shard_size = self.intermediate_size
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size)
# DBRX uses GLU for each experts.
# GLU has 3 linear layers: w1, v1 and w2.
if weight_name.endswith("w1"):
+42 -26
View File
@@ -942,11 +942,6 @@ class GptOssForCausalLM(nn.Module):
loaded_params: set[str] = set()
mxfp4_block = 32
moe_tp_rank = get_parallel().moe_tp_rank
moe_tp_size = get_parallel().moe_tp_size
moe_ep_rank = get_parallel().moe_ep_rank
moe_ep_size = get_parallel().moe_ep_size
intermediate_size = self.config.intermediate_size
original_intermediate_size = getattr(
self.config, "original_intermediate_size", intermediate_size
@@ -956,30 +951,49 @@ class GptOssForCausalLM(nn.Module):
)
intermediate_size_block = intermediate_size // mxfp4_block
per_rank_intermediate_size_block = math.ceil(
intermediate_size_block / moe_tp_size
)
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
# Calculate common slicing bounds for current rank
assert self.config.num_local_experts % moe_ep_size == 0
moe_num_global_experts = self.config.num_local_experts
moe_num_local_experts = self.config.num_local_experts // moe_ep_size
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
moe_tp_rank_end = min(
(moe_tp_rank + 1) * per_rank_intermediate_size, original_intermediate_size
)
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
weight_device = next(iter(params_dict.values())).device
for name, weight in weights:
weight = weight.to(weight_device)
if not any(
suffix in name
for suffix in (
"gate_up_proj_blocks",
"down_proj_blocks",
"gate_up_proj_scales",
"down_proj_scales",
"gate_up_proj_bias",
"down_proj_bias",
)
):
continue
experts = self.get_submodule(name.rsplit(".", 1)[0])
moe_tp_rank = experts.moe_tp_rank
moe_tp_size = experts.moe_tp_size
moe_ep_rank = experts.moe_ep_rank
moe_ep_size = experts.moe_ep_size
per_rank_intermediate_size_block = math.ceil(
intermediate_size_block / moe_tp_size
)
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
# Calculate common slicing bounds for current rank
assert self.config.num_local_experts % moe_ep_size == 0
moe_num_global_experts = self.config.num_local_experts
moe_num_local_experts = self.config.num_local_experts // moe_ep_size
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
moe_tp_rank_end = min(
(moe_tp_rank + 1) * per_rank_intermediate_size,
original_intermediate_size,
)
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
if "gate_up_proj_blocks" in name:
# Handle MLP gate and up projection weights
new_name = name.replace("gate_up_proj_blocks", "w13_weight")
@@ -1240,8 +1254,10 @@ class GptOssForCausalLM(nn.Module):
weight_loader = param.weight_loader
if "bias" not in name:
loaded_weight = loaded_weight.transpose(-2, -1)
if "w2_weight_bias" in name and get_parallel().moe_tp_rank != 0:
loaded_weight = loaded_weight.zero_()
if "w2_weight_bias" in name:
experts = self.get_submodule(name.rsplit(".", 1)[0])
if experts.moe_tp_rank != 0:
loaded_weight = loaded_weight.zero_()
weight_loader(
param,
+39 -33
View File
@@ -100,29 +100,20 @@ STACKED_DENSE_PARAMS_MAPPING = [
("down_proj", "down_proj", None),
]
# Online RL weight-sync streams routed experts one at a time as FULL (unsharded)
# per-expert tensors named `...mlp.experts.{j}.gate_proj/up_proj/down_proj.weight`.
# Disk checkpoints only ever carry the fused w13_weight/w2_weight, so this pattern
# never fires on the ordinary loading path.
# Online RL updates use full per-expert gate/up/down tensors; disk checkpoints
# contain fused w13/w2 weights and do not match this pattern.
_PER_EXPERT_WEIGHT_RE = re.compile(
r"^(?P<pfx>.+\.mlp\.experts)\.(?P<eid>\d+)\.(?P<proj>gate_proj|up_proj|down_proj)\.weight$"
)
def _shard_full_to_local(
loaded_weight: torch.Tensor, dst: torch.Tensor, dim: int
loaded_weight: torch.Tensor, dst: torch.Tensor, dim: int, *, tp_rank: int
) -> torch.Tensor:
"""Slice a FULL (unsharded) per-expert weight to this MoE-TP rank's shard along `dim`.
The online weight-sync ships full per-expert tensors (parallelism-agnostic HF
layout); sglang owns its own MoE-TP sharding, so narrow here per
get_parallel().moe_tp_rank. With TP1 the dims already match and this is the
identity, so the ordinary path is byte-for-byte unchanged.
"""
"""Shard full HF per-expert updates with the owning module's MoE TP rank."""
if loaded_weight.shape[dim] == dst.shape[dim]:
return loaded_weight
rank = get_parallel().moe_tp_rank
return loaded_weight.narrow(dim, rank * dst.shape[dim], dst.shape[dim])
return loaded_weight.narrow(dim, tp_rank * dst.shape[dim], dst.shape[dim])
KV_REPLICATED_SUFFIXES = (
@@ -1238,12 +1229,10 @@ class InklingForConditionalGeneration(nn.Module):
return False
param = params_dict[name]
if loaded_weight.shape != param.shape:
# shared experts shard over the full tp group; routed over moe_tp
tp_rank = (
get_parallel().tp_rank
if ".shared_experts" in name
else get_parallel().moe_tp_rank
)
# Both routed and shared modules own their constructed TP layout.
module = self.get_submodule(name.rsplit(".", 1)[0])
module = getattr(module, "base_layer", module)
tp_rank = module.moe_tp_rank
for dim in range(loaded_weight.ndim):
if loaded_weight.shape[dim] == param.shape[dim]:
continue
@@ -1297,10 +1286,8 @@ class InklingForConditionalGeneration(nn.Module):
hold only its contiguous slice (the fused loader shards the intermediate dim only).
No-op when EP is off or for replicated shared-expert tensors.
"""
ep_size = get_parallel().moe_ep_size
if (
ep_size <= 1
or ".experts." not in name
".experts." not in name
# per-expert RL sync tensors do their own EP remap in _load_per_expert_param;
# a full per-expert tensor whose dim 0 happens to equal n_routed_experts must
# not be pre-narrowed here.
@@ -1309,8 +1296,17 @@ class InklingForConditionalGeneration(nn.Module):
or loaded_weight.shape[0] != self.text_config.n_routed_experts
):
return loaded_weight
local = self.text_config.n_routed_experts // ep_size
start = get_parallel().moe_ep_rank * local
prefix = name.split(".experts.", 1)[0] + ".experts"
try:
module = self.get_submodule(prefix)
except AttributeError:
# Checkpoints may contain a layer absent from this model.
return loaded_weight
module = getattr(module, "base_layer", module)
if module.moe_ep_size <= 1:
return loaded_weight
local = self.text_config.n_routed_experts // module.moe_ep_size
start = module.moe_ep_rank * local
return loaded_weight.narrow(0, start, local).contiguous()
def _load_fused_moe_param(
@@ -1403,10 +1399,10 @@ class InklingForConditionalGeneration(nn.Module):
f"per-expert RL weight-sync does not support the trtllm MoE layout ({target}); "
"serve RL rollouts with the triton MoE runner"
)
ep_size = get_parallel().moe_ep_size
ep_size = moe.moe_ep_size
if ep_size > 1:
local = self.text_config.n_routed_experts // ep_size
first = get_parallel().moe_ep_rank * local
first = moe.moe_ep_rank * local
if not (first <= eid < first + local):
loaded_params.add(target) # another rank owns this expert
return True
@@ -1415,7 +1411,9 @@ class InklingForConditionalGeneration(nn.Module):
dst = params_dict[target].data[
eid
] # [H, I_local]; shard intermediate (dim 1)
dst.copy_(_shard_full_to_local(loaded_weight, dst, dim=1))
dst.copy_(
_shard_full_to_local(loaded_weight, dst, dim=1, tp_rank=moe.moe_tp_rank)
)
else:
w13 = params_dict[target].data[
eid
@@ -1429,7 +1427,9 @@ class InklingForConditionalGeneration(nn.Module):
dst = w13[idx * half : (idx + 1) * half] # contiguous [gate || up]
else:
dst = w13[idx::2] # Inkling-interleaved rows
dst.copy_(_shard_full_to_local(loaded_weight, dst, dim=0))
dst.copy_(
_shard_full_to_local(loaded_weight, dst, dim=0, tp_rank=moe.moe_tp_rank)
)
loaded_params.add(target)
return True
@@ -1590,18 +1590,21 @@ class InklingForConditionalGeneration(nn.Module):
continue
if ".experts.w13_weight" in name:
experts = modules_dict.get(name.rsplit(".", 1)[0])
experts = getattr(experts, "base_layer", experts)
# bf16 routed layers run the stock FusedMoE forward (not moe_tp_forward)
# under --enable-lora, or natively on trtllm_routed for UNQUANTIZED
# checkpoints: de-interleave per moe_tp block for the stock weight prep.
if (
loaded_weight.dtype != torch.uint8
experts is not None
and loaded_weight.dtype != torch.uint8
and self.text_config.inference_moe_w13_interleaved
and (
lora_compatible_layout_enabled()
or bf16_routed_uses_stock_fused_moe(self.quant_config)
)
):
tp = get_parallel().moe_tp_size
tp = experts.moe_tp_size
n_e, two_f, hid = loaded_weight.shape
loaded_weight = deinterleave_gate_up(
loaded_weight.view(n_e, tp, two_f // tp, hid), dim=2
@@ -1625,16 +1628,19 @@ class InklingForConditionalGeneration(nn.Module):
):
continue
if ".shared_experts.shared_w13_weight" in name:
experts = modules_dict.get(name.rsplit(".", 1)[0])
experts = getattr(experts, "base_layer", experts)
# InklingSharedFusedMoE's bf16 path needs contiguous [gate||up] w13 for the
# SRT runner's silu_and_mul, unlike the interleaved bmm/moe_tp_forward paths.
# Per-rank blocks are sized by the FULL tp group (InklingSharedFusedMoE always
# shards over it at EP=1), NOT moe_tp (= tp/ep, wrong under --ep-size > 1).
if (
loaded_weight.dtype != torch.uint8
experts is not None
and loaded_weight.dtype != torch.uint8
and self.text_config.inference_moe_w13_interleaved
and use_inkling_shared_fused_moe()
):
tp = get_parallel().tp_size
tp = experts.moe_tp_size
n_e, two_f, hid = loaded_weight.shape
loaded_weight = deinterleave_gate_up(
loaded_weight.view(n_e, tp, two_f // tp, hid), dim=2
+1 -1
View File
@@ -1739,7 +1739,7 @@ class KimiK3DeltaAttention(nn.Module):
def _a_log_weight_loader(
param: torch.Tensor, loaded_weight: torch.Tensor
) -> None:
tp_rank = get_parallel().attn_tp_rank
tp_rank = self.attn_tp_rank
shard_size = param.data.shape[2] # local_num_heads
start_idx = tp_rank * shard_size
+4 -6
View File
@@ -173,6 +173,7 @@ class MultiHeadRMSNorm(nn.Module):
) -> None:
super().__init__()
self.tp_world = get_parallel().attn_tp_size
self.tp_rank = get_parallel().attn_tp_rank
self.num_heads = num_heads
self.num_heads_per_tp = num_heads // self.tp_world
self.head_dim = head_dim
@@ -183,16 +184,13 @@ class MultiHeadRMSNorm(nn.Module):
self.apply_layernorm_1p = apply_layernorm_1p
self.variance_epsilon = eps
@staticmethod
def weight_loader(
self,
param: nn.Parameter,
loaded_weight: torch.Tensor,
) -> None:
tp_world = get_parallel().attn_tp_size
tp_rank = get_parallel().attn_tp_rank
shard_size = loaded_weight.shape[0] // tp_world
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
shard_size = loaded_weight.shape[0] // self.tp_world
shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size)
param.data.copy_(loaded_weight[shard].reshape_as(param))
def forward(
@@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
import unittest
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock
import torch
@@ -31,6 +31,7 @@ class _RecordingLayer:
def __init__(self, is_gated: bool):
self.moe_runner_config = MoeRunnerConfig(is_gated=is_gated)
self.moe_tp_size = 1
self.params = {}
def register_parameter(self, name, param):
@@ -47,20 +48,18 @@ def _create_weights(is_gated: bool, block_quant: bool):
is_checkpoint_fp8_serialized=False,
)
with patch.object(fp8_quant, "get_parallel") as parallel:
parallel.return_value.tp_size = 1
fp8_quant.Fp8MoEMethod.create_fp8_moe_weight_(
layer=layer,
num_experts=NUM_EXPERTS,
hidden_size=HIDDEN,
intermediate_size_per_partition=INTERMEDIATE,
block_quant=block_quant,
quant_config=quant_config,
use_mxfp8=False,
is_checkpoint_fp8_serialized=False,
is_fp4_expert=False,
params_dtype=torch.bfloat16,
)
fp8_quant.Fp8MoEMethod.create_fp8_moe_weight_(
layer=layer,
num_experts=NUM_EXPERTS,
hidden_size=HIDDEN,
intermediate_size_per_partition=INTERMEDIATE,
block_quant=block_quant,
quant_config=quant_config,
use_mxfp8=False,
is_checkpoint_fp8_serialized=False,
is_fp4_expert=False,
params_dtype=torch.bfloat16,
)
return layer.params
@@ -12,7 +12,6 @@ import torch
from sglang.srt.layers.quantization.blockwise_int8 import BlockInt8Config
from sglang.srt.layers.quantization.w8a8_int8 import W8A8Int8Config
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_device_sm
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.layer_ut_utils import (
@@ -113,9 +112,7 @@ class TestBlockInt8Linear(_Int8LinearCheck):
activation_scheme="dynamic",
weight_block_size=[128, 128],
)
# create_weights reads get_parallel().tp_size, not the layer's argument.
with get_parallel().override(tp_size=1, tp_rank=0):
layer = make_tp1_column_parallel_linear(quant_config, n, k)
layer = make_tp1_column_parallel_linear(quant_config, n, k)
w = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) / 10
w_int8, scale_inv, w_dequant = _quantize_int8_block(w)
load_linear_weights(layer, weight=w_int8, weight_scale_inv=scale_inv)
@@ -0,0 +1,162 @@
"""Auxiliary checkpoint shards stay with the constructed activation/norm layer."""
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
import torch
from torch import nn
from sglang.srt.layers.activation import ScaledActivation
from sglang.srt.models.bailing_moe_linear import BailingGroupRMSNormGate
from sglang.srt.models.commandr import LayerNorm
from sglang.srt.models.kimi_k3 import KimiK3DeltaAttention
from sglang.srt.models.minimax_m3 import MultiHeadRMSNorm
from sglang.srt.runtime_context import SpawnRanks, get_parallel, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.parallel_groups import parallel_scope, publish
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large")
KINDS = (
"scaled",
"scaled_replicated",
"command",
"command_replicated",
"bailing",
"minimax",
"minimax_1p",
"kimi",
)
def loading_scope(changed):
if not changed:
return nullcontext()
return parallel_scope(
tp_size=1,
tp_rank=0,
tp_group=None,
attn_tp_size=1,
attn_tp_rank=0,
attn_tp_group=None,
attn_dp_size=1,
attn_dp_rank=0,
attn_cp_size=1,
attn_cp_rank=0,
moe_tp_size=1,
moe_tp_rank=0,
moe_ep_size=1,
moe_ep_rank=0,
moe_ep_group=None,
moe_dp_size=1,
moe_dp_rank=0,
)
def build_layer(kind):
parallel = get_parallel()
if kind.startswith("scaled"):
replicated = kind.endswith("replicated")
module = ScaledActivation(nn.GELU(), 32, input_is_parallel=not replicated)
return (
module,
module.scales,
(32,),
0 if replicated else parallel.tp_rank,
1 if replicated else parallel.tp_size,
)
if kind.startswith("command"):
replicated = kind.endswith("replicated")
module = LayerNorm(16 if replicated else (8 // parallel.tp_size, 16))
return (
module,
module.weight,
(16,) if replicated else (8, 16),
0 if replicated else parallel.tp_rank,
1 if replicated else parallel.tp_size,
)
if kind == "bailing":
module = BailingGroupRMSNormGate(64 // parallel.attn_tp_size, group_size=8)
return (
module,
module.weight,
(64,),
parallel.attn_tp_rank,
parallel.attn_tp_size,
)
if kind.startswith("minimax"):
module = MultiHeadRMSNorm(8, 16, apply_layernorm_1p=kind.endswith("1p"))
return (
module,
module.weight,
(8, 16),
parallel.attn_tp_rank,
parallel.attn_tp_size,
)
config = SimpleNamespace(
dtype=torch.get_default_dtype(),
v_head_dim=8,
rms_norm_eps=1e-6,
linear_attn_config=dict(head_dim=8, num_heads=8, short_conv_kernel_size=3),
)
module = KimiK3DeltaAttention(0, 32, config)
return module, module.A_log, (8,), parallel.attn_tp_rank, parallel.attn_tp_size
def values(shape, param, offset=0):
count = 1
for size in shape:
count *= size
return ((torch.arange(count, device=param.device) + offset) % 17 + 1).reshape(
shape
).to(param.dtype) / 32
def load_layer(kind, *, changed=False, flat=False, old_kimi=False, offset=0):
module, param, shape, rank, size = build_layer(kind)
full = values(shape, param, offset)
checkpoint = full.flatten() if flat else full
if old_kimi:
checkpoint = full.view(1, 1, 8, 1)
expected = full.chunk(size, dim=0)[rank].reshape_as(param)
with loading_scope(changed):
param.weight_loader(param, checkpoint)
torch.testing.assert_close(param, expected, rtol=0, atol=0)
return module, param, expected
class TestAuxiliaryLoaderLayouts(CustomTestCase):
def setUp(self):
reset_context()
self.addCleanup(reset_context)
def check_loads(self, changed):
for dp in (1, 2):
for rank in range(4):
reset_context()
publish(
ServerArgs(
model_path="dummy", device="cpu", tp_size=4, attn_dp_size=dp
),
role="test",
ranks=SpawnRanks(world_rank=rank),
)
for kind in KINDS:
for offset in (0, 7):
load_layer(kind, changed=changed, offset=offset)
if kind.startswith("minimax"):
load_layer(kind, changed=changed, flat=True, offset=offset)
if kind == "kimi":
load_layer(
kind, changed=changed, old_kimi=True, offset=offset
)
def test_loaders_after_scope_exit(self):
self.check_loads(True)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,116 @@
"""DBRX expert checkpoint shards follow the constructed TP layout."""
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
import torch
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
from sglang.srt.models.dbrx import DbrxExperts
from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=15, stage="nightly", runner_config="1-gpu-large")
def loading_scope(changed):
if not changed:
return nullcontext()
return get_parallel().override(
tp_size=1,
tp_rank=0,
tp_group=None,
attn_tp_size=1,
attn_tp_rank=0,
attn_tp_group=None,
attn_dp_size=1,
attn_dp_rank=0,
attn_cp_size=1,
attn_cp_rank=0,
moe_tp_size=1,
moe_tp_rank=0,
moe_ep_size=1,
moe_ep_rank=0,
moe_ep_group=None,
moe_dp_size=1,
moe_dp_rank=0,
)
def build_experts(dtype=torch.bfloat16):
config = SimpleNamespace(
d_model=32,
ffn_config=SimpleNamespace(moe_num_experts=4, moe_top_k=2, ffn_hidden_size=128),
)
module = DbrxExperts(config, 0, params_dtype=dtype)
assert isinstance(module.router.layer, ReplicatedLinear)
assert isinstance(module.router.layer.quant_method, UnquantizedLinearMethod)
return module
def values(shape, dtype, offset=0):
count = 1
for size in shape:
count *= size
return (
(((torch.arange(count, device="cuda") + offset) % 31 - 15) / 128)
.to(dtype)
.reshape(shape)
)
def load_experts(module, *, changed=False, flat=False, offset=0):
rank = get_parallel().tp_rank
start = rank * module.intermediate_size
stop = start + module.intermediate_size
loaded = {}
for index, name in enumerate(("w1", "v1", "w2")):
full = values((4, 128, 32), module.ws.dtype, offset + index * 7)
param = module.w2s if name == "w2" else module.ws
with loading_scope(changed):
param.weight_loader(param, full.flatten() if flat else full, name)
loaded[name] = full[:, start:stop, :]
expected_ws = torch.cat((loaded["w1"], loaded["v1"]), dim=1)
expected_w2 = loaded["w2"].transpose(1, 2)
torch.testing.assert_close(module.ws, expected_ws, rtol=0, atol=0)
torch.testing.assert_close(module.w2s, expected_w2, rtol=0, atol=0)
return expected_ws, expected_w2
@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU")
class TestDbrxExpertLoaderLayout(CustomTestCase):
def setUp(self):
reset_context()
self.addCleanup(reset_context)
def check_loads(self, changed):
with torch.inference_mode():
for size in (1, 2, 4):
for rank in range(size):
reset_context()
publish(
ServerArgs(model_path="dummy", device="cuda", tp_size=size),
role="test",
ranks=SpawnRanks(world_rank=rank),
)
for dtype in (torch.float32, torch.bfloat16):
module = build_experts(dtype)
for flat in (False, True):
for offset in (0, 11):
load_experts(
module,
changed=changed,
flat=flat,
offset=offset,
)
def test_native_loader_after_scope_exit(self):
self.check_loads(True)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,344 @@
"""GPT-OSS packed expert checkpoints follow the destination expert layout."""
import re
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.quantization.mxfp4 import Mxfp4Config, Mxfp4MoEMethod
from sglang.srt.layers.quantization.quark.quark import QuarkConfig
from sglang.srt.layers.quantization.quark.schemes.quark_w4a8_mxfp4_moe import (
QuarkW4A8MXFp4MoE,
)
from sglang.srt.layers.quantization.quark.weights import (
_load_gptoss_quark_expert_weights,
)
from sglang.srt.models.gpt_oss import GptOssForCausalLM
from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large")
EXPERT_PATTERN = re.compile(
r"^(.*\.mlp\.experts)\.(\d+)\.(gate_up_proj|down_proj)\."
r"(weight|weight_scale|input_scale|bias)$"
)
def loading_scope(changed):
if not changed:
return nullcontext()
return get_parallel().override(
tp_size=1,
tp_rank=0,
tp_group=None,
attn_tp_size=1,
attn_tp_rank=0,
attn_tp_group=None,
attn_dp_size=1,
attn_dp_rank=0,
attn_cp_size=1,
attn_cp_rank=0,
moe_tp_size=1,
moe_tp_rank=0,
moe_ep_size=1,
moe_ep_rank=0,
moe_ep_group=None,
moe_dp_size=1,
moe_dp_rank=0,
)
def quark_config():
return QuarkConfig(
quant_config={
"packed_modules_mapping": {},
"layer_quant_config": {},
"layer_type_quant_config": {},
"global_quant_config": {
"weight": {
"dtype": "fp4",
"qscheme": "per_group",
"group_size": 32,
"is_dynamic": False,
"scale_format": "e8m0",
},
"input_tensors": {
"dtype": "fp8_e4m3",
"qscheme": "per_tensor",
"is_dynamic": False,
},
"output_tensors": None,
"bias": None,
},
},
is_prequantized=True,
)
def build_model(kind, intermediate=384, original=None, device="cuda"):
config = SimpleNamespace(
hidden_size=128,
intermediate_size=intermediate,
num_local_experts=4,
num_hidden_layers=1,
)
if original is not None:
config.original_intermediate_size = original
model = GptOssForCausalLM.__new__(GptOssForCausalLM)
torch.nn.Module.__init__(model)
model.config = config
model.quant_config = (
Mxfp4Config(is_checkpoint_mxfp4_serialized=True)
if kind == "mxfp4"
else quark_config()
if kind == "quark"
else None
)
# Quark's W4A8 runner is AMD-only. Keep native parameter construction and
# the checkpoint loader while avoiding runner creation in this CUDA test.
runner_scope = (
patch.object(QuarkW4A8MXFp4MoE, "create_moe_runner", return_value=None)
if kind == "quark"
else nullcontext()
)
with torch.device(device), runner_scope:
experts = FusedMoE(
num_experts=4,
top_k=2,
hidden_size=128,
intermediate_size=intermediate,
layer_id=0,
gemm1_alpha=1.702,
gemm1_clamp_limit=10.0,
quant_config=model.quant_config,
params_dtype=torch.bfloat16,
with_bias=True,
use_weight_loader_fused=kind == "unquantized",
reduce_results=False,
prefix="model.layers.0.mlp.experts",
)
if kind == "mxfp4":
assert isinstance(experts.quant_method, Mxfp4MoEMethod)
elif kind == "quark":
assert isinstance(experts.scheme, QuarkW4A8MXFp4MoE)
model.model = torch.nn.Module()
model.model.layers = torch.nn.ModuleList([torch.nn.Module()])
model.model.layers[0].mlp = torch.nn.Module()
model.model.layers[0].mlp.experts = experts
return model, experts
def values(shape, dtype, offset=0, device="cuda"):
count = 1
for size in shape:
count *= size
indices = torch.arange(count, device=device) + offset
data = (
(indices % 16 + (indices // 16 % 16) * 16)
if dtype == torch.uint8
else (indices % 31 - 15) / 64
)
return data.to(dtype).reshape(shape)
def load_experts(model, kind, *, changed=False, offset=0):
module = model.model.layers[0].mlp.experts
config = model.config
intermediate = getattr(
config, "original_intermediate_size", config.intermediate_size
)
hidden = config.hidden_size
device = module.w13_weight.device
source = {
"w13_weight": values(
(4, 2 * intermediate, hidden // 2), torch.uint8, offset, device
),
"w2_weight": values(
(4, hidden, intermediate // 2), torch.uint8, offset + 3, device
),
"w13_weight_scale": values(
(4, 2 * intermediate, hidden // 32), torch.uint8, offset + 5, device
),
"w2_weight_scale": values(
(4, hidden, intermediate // 32), torch.uint8, offset + 7, device
),
"w13_weight_bias": values(
(4, 2 * intermediate), module.w13_weight_bias.dtype, offset + 11, device
),
"w2_weight_bias": values(
(4, hidden), module.w2_weight_bias.dtype, offset + 13, device
),
}
if kind == "quark":
source["w13_input_scale"] = values((4,), torch.float32, offset + 17, device)
source["w2_input_scale"] = values((4,), torch.float32, offset + 19, device)
prefix = "model.layers.0.mlp.experts"
suffixes = (
("w13_weight", "gate_up_proj", "blocks"),
("w2_weight", "down_proj", "blocks"),
("w13_weight_scale", "gate_up_proj", "scales"),
("w2_weight_scale", "down_proj", "scales"),
("w13_weight_bias", "gate_up_proj", "bias"),
("w2_weight_bias", "down_proj", "bias"),
)
checkpoint = []
if kind == "mxfp4":
for param_name, projection, suffix in suffixes:
data = source[param_name]
if suffix == "blocks":
data = data.unflatten(-1, (-1, 16))
checkpoint.append((f"{prefix}.{projection}_{suffix}", data))
checkpoint.append((f"{prefix}.unused_tensor", torch.ones(1, device=device)))
else:
for expert in range(4):
for param_name, projection, suffix in suffixes:
suffix = {"blocks": "weight", "scales": "weight_scale"}.get(
suffix, suffix
)
checkpoint.append(
(
f"{prefix}.{expert}.{projection}.{suffix}",
source[param_name][expert],
)
)
for param_name, projection in (
("w13_input_scale", "gate_up_proj"),
("w2_input_scale", "down_proj"),
):
checkpoint.append(
(
f"{prefix}.{expert}.{projection}.input_scale",
source[param_name][expert],
)
)
checkpoint.append(("not-an-expert-weight", torch.ones(1, device=device)))
before = {name: param.detach().clone() for name, param in module.named_parameters()}
with loading_scope(changed):
loaded = (
model._load_mxfp4_experts_weights(iter(checkpoint))
if kind == "mxfp4"
else _load_gptoss_quark_expert_weights(
model, iter(checkpoint), EXPERT_PATTERN
)
)
width = (
(config.intermediate_size // 32 + module.moe_tp_size - 1) // module.moe_tp_size
) * 32
start = module.moe_tp_rank * width
stop = min(start + width, intermediate)
expert_start = module.moe_ep_rank * module.num_local_experts
expert_stop = expert_start + module.num_local_experts
expected = {}
for name, original_data in source.items():
data = original_data[expert_start:expert_stop]
target = before[name]
if name.startswith("w13") and "input_scale" not in name:
if kind == "mxfp4":
data = data[:, 2 * start : 2 * stop]
if data.dim() == 3:
target[:, : data.shape[1], : data.shape[2]] = data
else:
target[:, : data.shape[1]] = data
else:
gate = data[:, 0::2, ...][:, start:stop]
up = data[:, 1::2, ...][:, start:stop]
half = target.shape[1] // 2
if data.dim() == 3:
target[:, : gate.shape[1], : gate.shape[2]] = gate
target[:, half : half + up.shape[1], : up.shape[2]] = up
else:
target[:, : gate.shape[1]] = gate
target[:, half : half + up.shape[1]] = up
elif name == "w2_weight":
data = data[..., start // 2 : stop // 2]
target[:, : data.shape[1], : data.shape[2]] = data
elif name == "w2_weight_scale":
data = data[..., start // 32 : stop // 32]
target[:, : data.shape[1], : data.shape[2]] = data
elif name == "w2_weight_bias":
target[:, : data.shape[1]] = (
data if module.moe_tp_rank == 0 else torch.zeros_like(data)
)
else:
target.copy_(data)
torch.testing.assert_close(getattr(module, name), target, rtol=0, atol=0)
expected[name] = target
assert loaded == {f"{prefix}.{name}" for name in source}
return expected
@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU")
class TestGptOssExpertLoaderLayout(CustomTestCase):
def setUp(self):
reset_context()
self.addCleanup(reset_context)
def check_loads(self, changed):
with torch.inference_mode():
for ep_size in (1, 2, 4):
for rank in range(4):
reset_context()
publish(
ServerArgs(
model_path="dummy",
device="cuda",
tp_size=4,
ep_size=ep_size,
moe_runner_backend="triton",
),
role="test",
ranks=SpawnRanks(world_rank=rank),
)
for kind in ("mxfp4", "quark"):
for intermediate, original in (
((384, None), (288, None), (384, 288))
if kind == "mxfp4"
else ((384, None), (512, None))
):
model, _ = build_model(kind, intermediate, original)
for offset in (0, 11):
load_experts(
model, kind, changed=changed, offset=offset
)
def test_normal_down_projection_bias_uses_the_expert_rank(self):
with torch.inference_mode():
for rank in range(4):
reset_context()
publish(
ServerArgs(
model_path="dummy",
device="cuda",
tp_size=4,
moe_runner_backend="triton",
),
role="test",
ranks=SpawnRanks(world_rank=rank),
)
model, module = build_model("unquantized")
for changed in (False, True):
bias = values((4, 128), module.w2_weight_bias.dtype)
expected = bias.clone() if rank == 0 else torch.zeros_like(bias)
with loading_scope(changed):
model._load_normal_weights(
[("model.layers.0.mlp.experts.down_proj_bias", bias)],
is_nextn=False,
weight_name_mapping=None,
)
torch.testing.assert_close(
module.w2_weight_bias, expected, rtol=0, atol=0
)
def test_native_loaders_after_scope_exit(self):
self.check_loads(True)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,263 @@
"""Inkling checkpoint preprocessing follows its constructed expert modules."""
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock, patch
import torch
from torch import nn
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.lora.layers import BaseLayerWithLoRA
from sglang.srt.models import inkling
from sglang.srt.models.inkling_common.moe import InklingSharedFusedMoE
from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
def values(shape, offset=0, *, device="cpu", dtype=torch.float32):
count = 1
for size in shape:
count *= size
return (
(((torch.arange(count, device=device) + offset) % 29 - 14) / 128)
.reshape(shape)
.to(dtype)
)
def loading_scope(changed):
if not changed:
return nullcontext()
return get_parallel().override(
tp_size=1,
tp_rank=0,
tp_group=None,
attn_tp_size=1,
attn_tp_rank=0,
attn_tp_group=None,
attn_dp_size=1,
attn_dp_rank=0,
attn_cp_size=1,
attn_cp_rank=0,
moe_tp_size=1,
moe_tp_rank=0,
moe_ep_size=1,
moe_ep_rank=0,
moe_ep_group=None,
moe_dp_size=1,
moe_dp_rank=0,
)
def build_model(*, width=32, intermediate=64, wrapped=False):
model = inkling.InklingForConditionalGeneration.__new__(
inkling.InklingForConditionalGeneration
)
nn.Module.__init__(model)
model.text_config = SimpleNamespace(
n_routed_experts=8, inference_moe_w13_interleaved=True
)
model.quant_config = model.audio = model.visual = None
model.llm = nn.Module()
block = nn.Module()
block.mlp = nn.Module()
routed = FusedMoE(
num_experts=8,
hidden_size=width,
intermediate_size=intermediate,
layer_id=0,
top_k=1,
use_weight_loader_fused=True,
)
shared = InklingSharedFusedMoE(
2, width, intermediate, 0, "llm.layers.0.mlp.shared_experts", None, True
)
block.mlp.experts = BaseLayerWithLoRA(routed, Mock()) if wrapped else routed
block.mlp.shared_experts = shared
model.llm.layers = nn.ModuleList([block])
return model, routed, shared
def load_per_expert(model, routed, *, changed=False, contiguous=False, offset=0):
width = routed.hidden_size
intermediate = routed.intermediate_size_per_partition * routed.moe_tp_size
local = 8 // routed.moe_ep_size
start = routed.moe_ep_rank * local
shard = routed.moe_tp_rank * routed.intermediate_size_per_partition
weights = []
gates = []
ups = []
downs = []
for expert in range(8):
gate = values(
(intermediate, width),
offset + 3 * expert,
device=routed.w13_weight.device,
dtype=routed.w13_weight.dtype,
)
up = values(
(intermediate, width),
offset + 3 * expert + 1,
device=gate.device,
dtype=gate.dtype,
)
down = values(
(width, intermediate),
offset + 3 * expert + 2,
device=gate.device,
dtype=gate.dtype,
)
for name, weight in (("gate_proj", gate), ("up_proj", up), ("down_proj", down)):
weights.append(
(f"model.llm.layers.0.mlp.experts.{expert}.{name}.weight", weight)
)
if start <= expert < start + local:
gates.append(gate[shard : shard + routed.intermediate_size_per_partition])
ups.append(up[shard : shard + routed.intermediate_size_per_partition])
downs.append(
down[:, shard : shard + routed.intermediate_size_per_partition]
)
with (
loading_scope(changed),
patch.object(
inkling, "lora_compatible_layout_enabled", return_value=contiguous
),
):
loaded = model.load_weights(weights)
gate, up = torch.stack(gates), torch.stack(ups)
expected_w13 = (
torch.cat((gate, up), dim=1)
if contiguous
else torch.stack((gate, up), dim=2).flatten(1, 2)
)
expected_w2 = torch.stack(downs)
suffix = (
".base_layer"
if isinstance(model.llm.layers[0].mlp.experts, BaseLayerWithLoRA)
else ""
)
assert loaded == {
f"llm.layers.0.mlp.experts{suffix}.w13_weight",
f"llm.layers.0.mlp.experts{suffix}.w2_weight",
}
torch.testing.assert_close(routed.w13_weight, expected_w13, rtol=0, atol=0)
torch.testing.assert_close(routed.w2_weight, expected_w2, rtol=0, atol=0)
return expected_w13, expected_w2
def load_fused(model, routed, shared, *, changed=False, offset=0):
weights = []
expected = []
for kind, module, experts in (
("experts", routed, 8),
("shared_experts", shared, 2),
):
width = module.hidden_size
intermediate = module.intermediate_size_per_partition * module.moe_tp_size
w13 = values(
(experts, 2 * intermediate, width),
offset,
device=module.w13_weight.device,
dtype=module.w13_weight.dtype,
)
w2 = values(
(experts, width, intermediate),
offset + 1,
device=w13.device,
dtype=w13.dtype,
)
prefix = f"model.llm.layers.0.mlp.{kind}."
weights.extend(
(
(
prefix
+ (
"shared_w13_weight"
if kind == "shared_experts"
else "w13_weight"
),
w13,
),
(
prefix
+ ("shared_w2_weight" if kind == "shared_experts" else "w2_weight"),
w2,
),
)
)
local = experts // module.moe_ep_size
first = module.moe_ep_rank * local
rank = module.moe_tp_rank
local_w13 = w13[first : first + local].chunk(module.moe_tp_size, dim=1)[rank]
local_w13 = torch.cat((local_w13[:, 0::2], local_w13[:, 1::2]), dim=1)
local_w2 = w2[first : first + local].chunk(module.moe_tp_size, dim=2)[rank]
expected.append((module, local_w13, local_w2))
weights.append(("model.llm.layers.999.mlp.experts.w13_weight", weights[0][1]))
with (
loading_scope(changed),
patch.object(inkling, "lora_compatible_layout_enabled", return_value=True),
patch.object(inkling, "use_inkling_shared_fused_moe", return_value=True),
):
loaded = model.load_weights(weights)
assert loaded == {
f"llm.layers.0.mlp.{kind}.{leaf}"
for kind in ("experts", "shared_experts")
for leaf in ("w13_weight", "w2_weight")
}
for module, w13, w2 in expected:
torch.testing.assert_close(module.w13_weight, w13, rtol=0, atol=0)
torch.testing.assert_close(module.w2_weight, w2, rtol=0, atol=0)
return expected
class TestInklingMoeLoaderLayout(CustomTestCase):
def setUp(self):
reset_context()
self.addCleanup(reset_context)
def check_layouts(self, *, changed=False, wrapped=False):
for ep in (1, 2, 4):
for rank in range(4):
publish(
ServerArgs(
model_path="dummy",
device="cpu",
tp_size=4,
ep_size=ep,
moe_runner_backend="triton",
),
role="test",
ranks=SpawnRanks(world_rank=rank),
)
for contiguous in (False, True):
model, routed, _ = build_model(wrapped=wrapped)
for offset in (0, 7):
load_per_expert(
model,
routed,
changed=changed,
contiguous=contiguous,
offset=offset,
)
if not wrapped:
model, routed, shared = build_model()
for offset in (0, 7):
load_fused(
model, routed, shared, changed=changed, offset=offset
)
def test_native_expert_loaders_after_scope_exit(self):
self.check_layouts(changed=True)
def test_native_per_expert_loader_with_lora_parameter_aliases(self):
self.check_layouts(changed=True, wrapped=True)
if __name__ == "__main__":
unittest.main()
@@ -20,7 +20,7 @@ import unittest
import torch
import sglang.srt.models.inkling as inkling_mod
from sglang.srt.runtime_context import reset_context
from sglang.srt.runtime_context import get_parallel, reset_context
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import publish_build_topology
@@ -38,6 +38,10 @@ class _FakeModel:
inference_moe_w13_interleaved=interleaved,
)
self._moe = moe if moe is not None else types.SimpleNamespace()
self._moe.moe_ep_size = get_parallel().moe_ep_size
self._moe.moe_ep_rank = get_parallel().moe_ep_rank
self._moe.moe_tp_size = get_parallel().moe_tp_size
self._moe.moe_tp_rank = get_parallel().moe_tp_rank
def get_submodule(self, path):
return self._moe