[Refactor] moe: Scope shared-expert configuration to each model (#44913)

This commit is contained in:
Cheng Wan
2026-10-19 16:59:37 -07:00
committed by GitHub
parent 964ab2b280
commit 6b90e5716d
62 changed files with 1086 additions and 802 deletions
@@ -39,6 +39,12 @@ class Model(msgspec.Struct):
doc="Whether weight loading overlaps startup.",
)
# Runtime-only binding: loaders/runners scope the immutable model-owned value.
shared_experts = Derived(
fn="sglang.srt.configs.shared_experts.default_shared_experts_config",
doc="Shared-expert layout of the model being constructed or executed.",
)
# -------------------------------------------------------------------------
# Model and tokenizer
# -------------------------------------------------------------------------
@@ -0,0 +1,21 @@
"""Resolved shared-expert layout for one loaded model."""
import msgspec
class SharedExpertsConfig(msgspec.Struct, frozen=True):
"""Immutable layout shared by a model, its routers and its captured graphs."""
fusion_disabled: bool = False
num_fused_shared_experts: int = 0
use_waterfill: bool = False
@property
def num_shared_topk_slots(self) -> int:
"""Shared id/weight slots emitted before Waterfill expansion."""
return 0 if self.use_waterfill else self.num_fused_shared_experts
def default_shared_experts_config(args) -> SharedExpertsConfig:
"""Publication has no loaded model yet; retain only the startup intent."""
return SharedExpertsConfig(fusion_disabled=args.disable_shared_experts_fusion)
+13 -12
View File
@@ -24,9 +24,8 @@ from sglang.srt.layers.moe.topk import (
)
from sglang.srt.layers.moe.utils import (
has_per_rank_fused_shared_slots,
shared_topk_slots,
waterfill_routes_shared_experts,
)
from sglang.srt.runtime_context import get_model
from sglang.srt.utils import is_hip, is_npu, is_xpu
logger = logging.getLogger(__name__)
@@ -41,7 +40,7 @@ class HashTopK(nn.Module):
self,
topk,
num_experts,
num_fused_shared_experts,
*,
vocab_size,
scoring_func="sqrtsoftplus",
routed_scaling_factor=1.5,
@@ -51,20 +50,18 @@ class HashTopK(nn.Module):
super().__init__()
self.layer_id = layer_id
self.enable_waterfill = waterfill_routes_shared_experts(
num_fused_shared_experts
)
self.shared_experts = get_model().shared_experts
self.enable_waterfill = self.shared_experts.use_waterfill
self.waterfill_balancer = None
num_routed_experts_per_token = topk - num_fused_shared_experts
self.num_experts = num_experts
# Trailing columns the hash router fills with fused shared experts.
self.num_shared_topk_slots = shared_topk_slots(num_fused_shared_experts)
self.topk = num_routed_experts_per_token + self.num_shared_topk_slots
# topk counts routed experts; the trailing columns the hash router fills
# with fused shared experts follow them.
self.topk = topk + self.num_shared_topk_slots
self.routed_scaling_factor = routed_scaling_factor
self.score_func = scoring_func
self.tid2eid = nn.Parameter(
torch.empty(vocab_size, num_routed_experts_per_token, dtype=torch.int32),
torch.empty(vocab_size, topk, dtype=torch.int32),
requires_grad=False,
)
self._init_default_tid2eid()
@@ -78,6 +75,10 @@ class HashTopK(nn.Module):
"with fused shared experts; pass --disable-shared-experts-fusion."
)
@property
def num_shared_topk_slots(self) -> int:
return self.shared_experts.num_shared_topk_slots
def _init_default_tid2eid(self) -> None:
topk = self.tid2eid.shape[1]
if topk == 0:
@@ -286,7 +287,7 @@ class HashTopK(nn.Module):
num_physical_routed_experts,
TopKConfig(
top_k=self.topk,
num_shared_topk_slots=num_shared_topk_slots,
shared_experts=self.shared_experts,
routed_scaling_factor=self.routed_scaling_factor,
apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output,
),
+15 -17
View File
@@ -38,10 +38,12 @@ import triton.language as tl
if TYPE_CHECKING:
from triton_kernels.tensor_details.ragged_tensor import RaggedTensorMetadata
from sglang.srt.configs.shared_experts import SharedExpertsConfig
from sglang.srt.runtime_context import (
get_exec,
get_forward,
get_lora,
get_model,
get_parallel,
get_server_args,
)
@@ -106,8 +108,6 @@ from sglang.srt.layers.moe import (
)
from sglang.srt.layers.moe.utils import (
has_per_rank_fused_shared_slots,
shared_topk_slots,
waterfill_routes_shared_experts,
)
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
from sglang.srt.utils import (
@@ -258,9 +258,7 @@ class TopKConfig:
topk_group: Optional[int] = None
num_expert_group: Optional[int] = None
renormalize: bool = True
# Trailing columns of top_k that the router fills with fused shared experts.
# Zero when Waterfill appends the shared expert after routing.
num_shared_topk_slots: int = 0
shared_experts: SharedExpertsConfig = SharedExpertsConfig()
custom_routing_function: Optional[Callable] = None
correction_bias: Optional[torch.Tensor] = None
torch_native: bool = False
@@ -285,6 +283,10 @@ class TopKConfig:
default=None, init=False, repr=False, compare=False
)
@property
def num_shared_topk_slots(self) -> int:
return self.shared_experts.num_shared_topk_slots
def correction_bias_for_dtype(self, dtype: torch.dtype) -> Optional[torch.Tensor]:
"""Return correction bias in ``dtype``, reusing a per-TopK lazy copy."""
correction_bias = self.correction_bias
@@ -588,12 +590,11 @@ def _simulate_balanced_routing(
class TopK(BaseFusedOp):
"""
Parameters:
--top_k: The all number of top experts selected per token, including the fused shared expert(s).
--top_k: the number of routed experts selected per token. When the model being built
fuses shared experts (``get_num_fused_shared_experts``), their columns follow: written
by the router (``TopKConfig.num_shared_topk_slots``), or appended by Waterfill.
--layer_id: global index of the owning layer. Required: routed-experts capture and
per-layer EPLB state are keyed by it.
--num_fused_shared_experts: shared experts the experts layer runs as extra expert slots.
Their trailing top-k columns are written by the router, or by Waterfill when it is
enabled (``TopKConfig.num_shared_topk_slots`` then excludes them).
--routed_scaling_factor: the scaling factor for routed experts in topk_weights.
--fused_shared_experts_scaling_factor: absolute appended weight on the global AITER
path; otherwise multiplies the router's existing shared weight under standard EP.
@@ -609,7 +610,6 @@ class TopK(BaseFusedOp):
topk_group: Optional[int] = None,
num_expert_group: Optional[int] = None,
renormalize: bool = True,
num_fused_shared_experts: int = 0,
custom_routing_function: Optional[Callable] = None,
scoring_func: str = "softmax",
correction_bias: Optional[torch.Tensor] = None,
@@ -632,23 +632,21 @@ class TopK(BaseFusedOp):
self.layer_id = layer_id
self.enable_waterfill = waterfill_routes_shared_experts(
num_fused_shared_experts
)
shared_experts = get_model().shared_experts
self.enable_waterfill = shared_experts.use_waterfill
self.waterfill_balancer = None
num_shared_slots = shared_topk_slots(num_fused_shared_experts)
num_routed_experts_per_token = top_k - num_fused_shared_experts
num_shared_slots = shared_experts.num_shared_topk_slots
# Under the flashinfer_mxfp4 backend, fp4-expert ckpts take STANDARD
# (consumes topk_ids/weights); otherwise BYPASSED. No-op on other backends.
self.is_fp4_experts = is_fp4_experts
self.topk_config = TopKConfig(
top_k=num_routed_experts_per_token + num_shared_slots,
top_k=top_k + num_shared_slots,
use_grouped_topk=use_grouped_topk,
renormalize=renormalize,
topk_group=topk_group,
num_expert_group=num_expert_group,
num_shared_topk_slots=num_shared_slots,
shared_experts=shared_experts,
custom_routing_function=custom_routing_function,
correction_bias=correction_bias,
routed_scaling_factor=routed_scaling_factor,
+18 -71
View File
@@ -9,6 +9,7 @@ from typing import NamedTuple
import torch
from sglang.srt.configs.shared_experts import SharedExpertsConfig
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -500,12 +501,6 @@ def initialize_moe_config():
moe.tbo_token_distribution_threshold = overlap.tbo_token_distribution_threshold
moe.disable_fp4_allgather = exec_moe.disable_flashinfer_cutlass_moe_fp4_allgather
moe.quantization = get_model().quantization
# Seeded with the user's intent; each model's gate refines the ACTIVE
# value for its own build (install_shared_experts_fusion_decision).
moe.disable_shared_experts_fusion = exec_moe.disable_shared_experts_fusion
moe.speculative_disable_shared_experts_fusion = (
exec_moe.disable_shared_experts_fusion
)
def get_moe_a2a_backend() -> MoeA2ABackend:
@@ -543,29 +538,11 @@ def get_speculative_moe_a2a_backend() -> MoeA2ABackend:
def is_shared_experts_fusion_disabled() -> bool:
"""The ACTIVE shared-experts-fusion decision for the model being built.
return get_model().shared_experts.fusion_disabled
Written (both ways) by each MoE model's gate before its layers construct;
falls back to the config intent when no gate has run (models without an
auto-disable gate read the intent directly off the bag instead).
Construction-time only: a forward reads what its build baked in
(``num_fused_shared_experts`` on the layer). During a draft's build this
flag holds the DRAFT's decision, so a forward-time read would race the
build window — refuse it loudly."""
from sglang.srt.model_executor.forward_context import has_forward_context
if has_forward_context():
raise AssertionError(
"is_shared_experts_fusion_disabled() called inside a forward: the "
"fusion decision is construction-time state (it can hold the draft's "
"value while a draft builds). Read the value your build baked in, "
"e.g. the layer's num_fused_shared_experts."
)
moe = get_flags().moe
if moe.disable_shared_experts_fusion is None:
return get_exec().moe.disable_shared_experts_fusion
return moe.disable_shared_experts_fusion
def get_num_fused_shared_experts() -> int:
return get_model().shared_experts.num_fused_shared_experts
@contextmanager
@@ -574,9 +551,6 @@ def draft_model_build_scope():
settings; every value below returns to the target's on exit, including on
error.
- Shared-experts fusion: the gates the draft runs also record their
decision on the speculative leaf; the target's ACTIVE
``disable_shared_experts_fusion`` is restored.
- ``boundary_reduction`` is ``--speculative-boundary-reduction``.
Boundaries built here keep the resolved value after exit.
- ``enable_w4a4_mxfp4_megamoe`` is ``--speculative-enable-w4a4-mxfp4-megamoe``
@@ -588,7 +562,6 @@ def draft_model_build_scope():
whole lifecycle (build + capture + forward), which not every worker does.
"""
moe = get_flags().moe
original_fusion = moe.disable_shared_experts_fusion
original_scope = moe.in_speculative_scope
try:
moe.in_speculative_scope = True
@@ -603,7 +576,6 @@ def draft_model_build_scope():
yield
finally:
moe.in_speculative_scope = original_scope
moe.disable_shared_experts_fusion = original_fusion
def _draft_w4a4_mxfp4_megamoe_override() -> AbstractContextManager:
@@ -615,36 +587,29 @@ def _draft_w4a4_mxfp4_megamoe_override() -> AbstractContextManager:
return get_exec().moe.override(enable_w4a4_mxfp4_megamoe=draft_w4a4)
def install_shared_experts_fusion_decision(
def resolve_shared_experts_config(
model_class, hf_config, quant_config
) -> None:
"""Decide whether this runner's model fuses its shared experts, and install
the answer for the model it is about to build.
Called from the loader's single model-instantiation point, so the decision
is made once per runner — before any layer exists — and the model classes
are pure readers (``is_shared_experts_fusion_disabled``). A model family
that can auto-disable exposes the conditions as
``shared_experts_fusion_disable_reason(hf_config, quant_config)``; families
without one follow the user's intent.
Inside ``draft_model_build_scope`` the answer also lands on the speculative
leaf, so a flags dump afterwards shows both runners' decisions.
"""
) -> SharedExpertsConfig:
"""Resolve a model layout before construction; fusion requires both hooks."""
disabled = get_exec().moe.disable_shared_experts_fusion
gate = getattr(model_class, "shared_experts_fusion_disable_reason", None)
if not disabled:
gate = getattr(model_class, "shared_experts_fusion_disable_reason", None)
reason = gate(hf_config, quant_config) if gate is not None else None
if reason:
log_info_on_rank0(
logger, f"{reason} Shared experts fusion optimization is disabled."
)
disabled = True
moe = get_flags().moe
moe.disable_shared_experts_fusion = disabled
if moe.in_speculative_scope:
moe.speculative_disable_shared_experts_fusion = disabled
count_fusable = getattr(model_class, "num_fusable_shared_experts", None)
if disabled or gate is None or count_fusable is None:
num_fused = 0
else:
num_fused = count_fusable(hf_config)
return SharedExpertsConfig(
fusion_disabled=disabled,
num_fused_shared_experts=num_fused,
use_waterfill=num_fused > 0 and get_exec().moe.enable_waterfill,
)
def get_deepep_mode() -> DeepEPMode:
@@ -699,24 +664,6 @@ def has_per_rank_fused_shared_slots(num_shared: int) -> bool:
return num_shared > 0 and uses_per_rank_fused_shared_slots()
def waterfill_routes_shared_experts(num_fused_shared_experts: int) -> bool:
"""Whether Waterfill, not the top-k router, writes the fused shared experts' column.
Waterfill appends that column after the EPLB remap and may place it on another
rank, so the router it follows emits the routed columns only."""
return num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill
def shared_topk_slots(num_fused_shared_experts: int) -> int:
"""How many trailing top-k columns the router fills with fused shared experts.
``num_fused_shared_experts`` counts the shared experts the experts layer runs as
extra expert slots; the router writes their columns unless Waterfill does."""
if waterfill_routes_shared_experts(num_fused_shared_experts):
return 0
return num_fused_shared_experts
def is_flashinfer_cutedsl_v1_path() -> bool:
"""CuteDSL v1 + DeepEP low-latency path (no MoeRunner, no autotune)."""
return (
@@ -42,6 +42,7 @@ from sglang.srt.runtime_context import (
get_exec,
get_flags,
get_lora,
get_model,
get_parallel,
get_spec,
)
@@ -734,6 +735,12 @@ class CPUGraphRunner:
return is_bs_supported
def capture(self) -> None:
with get_model().override(
shared_experts=self.model_runner.model.shared_experts_config
):
self._capture()
def _capture(self) -> None:
capture_range = (
tqdm.tqdm(list(reversed(self.capture_bs)))
if get_parallel().tp_rank == 0
@@ -1794,7 +1794,10 @@ class ModelRunner:
forward_batch.split_index + forward_count,
self.model_config.num_hidden_layers,
)
with device_timer_ctx(self.device_timer, "split_prefill"):
with (
get_model().override(shared_experts=self.model.shared_experts_config),
device_timer_ctx(self.device_timer, "split_prefill"),
):
ret = self.model.forward_split_prefill(
forward_batch.input_ids,
forward_batch.positions,
@@ -1968,7 +1971,10 @@ class ModelRunner:
ctx_mgr = contextlib.nullcontext()
else:
ctx_mgr = forward_context(ForwardContext(attn_backend=self.attn_backend))
with ctx_mgr:
with (
get_model().override(shared_experts=self.model.shared_experts_config),
ctx_mgr,
):
mode_check = (
forward_batch.forward_mode.is_cpu_graph
if self.device == "cpu"
@@ -51,6 +51,7 @@ from sglang.srt.runtime_context import (
get_disagg,
get_exec,
get_flags,
get_model,
get_parallel,
)
from sglang.srt.speculative.spec_info import create_dummy_verify_input
@@ -243,7 +244,17 @@ class BaseRunner(ABC):
self.attn_tp_size = get_parallel().attn_tp_size
self.tbo_plugin = TboCudaGraphRunnerPlugin()
def model_context(self):
"""Select the layout owned by this runner's current model."""
return get_model().override(
shared_experts=self.model_runner.model.shared_experts_config
)
def warmup(self) -> None:
with self.model_context():
self._warmup()
def _warmup(self) -> None:
"""Run kernel warmup + autotune once, gated by mr._kernel_warmed_up."""
mr = self.model_runner
if getattr(mr, "_kernel_warmed_up", False):
@@ -1107,6 +1107,10 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
return forward_batch, attn_backend, pp_proxy_tensors
def capture(self) -> None:
with self.model_context():
self._capture()
def _capture(self) -> None:
# Warm up + autotune kernels once before capture (run-once across the
# decode + prefill runners; see BaseRunner.warmup).
self.warmup()
@@ -1593,7 +1597,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
shared_read_ends = self._resolve_shared_read_ends(
self._replay_attn_backend(), forward_batch.forward_mode
)
with timer_ctx, self.backend.replay_session():
with self.model_context(), timer_ctx, self.backend.replay_session():
self.load_batch(forward_batch, pp_proxy_tensors)
if envs.SGLANG_LOG_DECODE_GRAPH_KEY.get():
logger.info(
@@ -209,20 +209,21 @@ class EagerRunner(BaseRunner):
def execute(
self, forward_batch: ForwardBatch, pp_proxy_tensors=None, **kwargs
) -> Any:
mode = forward_batch.forward_mode
if mode.is_mixed() and not is_npu() and get_cp_strategy() is None:
# A mixed batch is extend-shaped (decode tails are 1-token
# extends); run it as EXTEND. NPU keeps MIXED for its dedicated
# kernel; CP keeps it to skip the zigzag split.
forward_batch.forward_mode = ForwardMode.EXTEND
mode = ForwardMode.EXTEND
if mode.is_decode():
return self._execute_decode(forward_batch, pp_proxy_tensors)
if mode.is_idle():
return self._execute_idle(forward_batch, pp_proxy_tensors)
if mode.is_extend(include_draft_extend_v2=True):
return self._execute_extend(forward_batch, pp_proxy_tensors)
raise ValueError(f"Invalid forward mode for eager runner: {mode}")
with self.model_context():
mode = forward_batch.forward_mode
if mode.is_mixed() and not is_npu() and get_cp_strategy() is None:
# A mixed batch is extend-shaped (decode tails are 1-token
# extends); run it as EXTEND. NPU keeps MIXED for its dedicated
# kernel; CP keeps it to skip the zigzag split.
forward_batch.forward_mode = ForwardMode.EXTEND
mode = ForwardMode.EXTEND
if mode.is_decode():
return self._execute_decode(forward_batch, pp_proxy_tensors)
if mode.is_idle():
return self._execute_idle(forward_batch, pp_proxy_tensors)
if mode.is_extend(include_draft_extend_v2=True):
return self._execute_extend(forward_batch, pp_proxy_tensors)
raise ValueError(f"Invalid forward mode for eager runner: {mode}")
def _resolve_decode_pdmux(
self,
@@ -1657,6 +1657,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
return forward_batch, self.model_runner.attn_backend
def capture(self) -> None:
with self.model_context():
self._capture()
def _capture(self) -> None:
# Warm up + autotune kernels once before capture (run-once across the
# decode + prefill runners; see BaseRunner.warmup).
self.warmup()
@@ -2307,7 +2311,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
self, forward_batch: ForwardBatch, **kwargs
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
self._validate_capture_hidden_mode(forward_batch)
with self.backend.replay_session():
with self.model_context(), self.backend.replay_session():
static_forward_batch = self.load_batch(forward_batch, **kwargs)
static_num_tokens = len(static_forward_batch.input_ids)
raw_num_tokens = self.raw_num_tokens
+6 -8
View File
@@ -86,7 +86,7 @@ from sglang.srt.distributed import (
from sglang.srt.layers.layer_boundary.stage import check_stage_producers
from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES
from sglang.srt.layers.moe.utils import (
install_shared_experts_fusion_decision,
resolve_shared_experts_config,
)
from sglang.srt.layers.quantization.base_config import (
QuantizationConfig,
@@ -285,11 +285,7 @@ def _initialize_model(
) -> nn.Module:
"""Initialize a model with the given configurations."""
model_class, _ = get_model_architecture(model_config)
# Decide the shared-experts-fusion question here, once per runner, before any
# layer exists: this is the only place a model class is instantiated, and it
# is the last point that still knows both the checkpoint's quantization and
# (through the build scope) whether this runner is a draft.
install_shared_experts_fusion_decision(
shared_experts = resolve_shared_experts_config(
model_class, model_config.hf_config, quant_config
)
kwargs = {
@@ -305,8 +301,10 @@ def _initialize_model(
if load_config.draft_model_idx is not None:
kwargs["draft_model_idx"] = load_config.draft_model_idx
model = model_class(**kwargs)
check_stage_producers(model)
with get_model().override(shared_experts=shared_experts):
model = model_class(**kwargs)
check_stage_producers(model)
model.shared_experts_config = shared_experts
return model
@@ -68,6 +68,10 @@ class BailingMoeV3VLForConditionalGeneration(nn.Module):
hf_config.text_config, quant_config
)
@staticmethod
def num_fusable_shared_experts(hf_config) -> int:
return BailingMoeV3ForCausalLM.num_fusable_shared_experts(hf_config.text_config)
def __init__(
self,
config: PretrainedConfig,
+10 -10
View File
@@ -32,6 +32,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -71,7 +72,6 @@ class BailingMoEModelNextN(nn.Module):
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
num_fused_shared_experts: int = 0,
) -> None:
super().__init__()
self.layer_group_size = 1
@@ -122,9 +122,6 @@ class BailingMoEModelNextN(nn.Module):
}
if _is_bailing_moe_v3_config(config):
decoder_layer_cls = BailingMoeV3DecoderLayer
decoder_kwargs["num_fused_shared_experts"] = (
num_fused_shared_experts
)
self.decoder = decoder_layer_cls(config, **decoder_kwargs)
else:
self.decoder = BailingMoEBlock(
@@ -215,6 +212,12 @@ class BailingMoeForCausalLMNextN(nn.Module):
expected_architecture="BailingMoeForCausalLMNextN",
)
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
if not _is_bailing_moe_v3_config(hf_config):
return 0
return BailingMoeV3ForCausalLM.num_fusable_shared_experts(hf_config)
def __init__(
self,
config: PretrainedConfig,
@@ -224,19 +227,16 @@ class BailingMoeForCausalLMNextN(nn.Module):
nn.Module.__init__(self)
self.config = config
self.quant_config = quant_config
self.num_fused_shared_experts = 0
# The weight loader reads this after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
is_bailing_moe_v3 = _is_bailing_moe_v3_config(config)
if is_bailing_moe_v3:
BailingMoeV3ForCausalLM.determine_num_fused_shared_experts(self)
elif hasattr(self, "determine_num_fused_shared_experts"):
# Asystem has determine_num_fused_shared_experts but theta does not.
self.determine_num_fused_shared_experts("BailingMoeForCausalLMNextN")
BailingMoeV3ForCausalLM.log_shared_experts_fusion(self)
self.model = BailingMoEModelNextN(
config,
quant_config,
prefix=add_prefix("model", prefix),
num_fused_shared_experts=self.num_fused_shared_experts,
)
self.lm_head = ParallelLMHead(
config.vocab_size,
+17 -25
View File
@@ -48,7 +48,7 @@ from sglang.srt.layers.moe import (
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
from sglang.srt.layers.multi_gate import (
create_multi_gate_mm_indices,
multi_gate_triton_kernel,
@@ -445,7 +445,6 @@ class BailingMoE(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
layer_id: int = 0,
prefix: str = "moe",
num_fused_shared_experts: int = 0,
alt_stream: Optional[torch.cuda.Stream] = None,
):
super().__init__()
@@ -466,7 +465,7 @@ class BailingMoE(nn.Module):
self.score_function = getattr(config, "score_function", None)
self.multi_gate = is_bailing_multi_gate_enabled(config)
self.num_fused_shared_experts = num_fused_shared_experts
self.num_fused_shared_experts = get_num_fused_shared_experts()
expert_swiglu_limit_list = getattr(config, "expert_swiglu_limit_list", None)
share_expert_swiglu_limit_list = getattr(
@@ -562,8 +561,7 @@ class BailingMoE(nn.Module):
# Non-A2A EP (standard): each GPU computes shared expert, outputs are summed
# via all_reduce → scale down by 1/ep_size to avoid double counting.
# Note: A2A EP (DeepEP) + fused shared experts is impossible (expert routing
# breaks for the extra shared expert ID), so it's auto-disabled in
# determine_num_fused_shared_experts().
# breaks for the extra shared expert ID), so the fusion veto disables it.
fused_shared_experts_scaling_factor = None
if self.moe_ep_size > 1 and self.num_fused_shared_experts > 0:
fused_shared_experts_scaling_factor = 1.0 / float(self.moe_ep_size)
@@ -581,7 +579,7 @@ class BailingMoE(nn.Module):
gemm1_clamp_limit=self.expert_swiglu_limit,
)
self.topk = TopK(
top_k=self.top_k + self.num_fused_shared_experts,
top_k=self.top_k,
layer_id=self.layer_id,
use_grouped_topk=self.use_grouped_topk,
renormalize=self.norm_expert_prob,
@@ -594,7 +592,6 @@ class BailingMoE(nn.Module):
apply_routed_scaling_factor_on_output=(
self.experts.should_fuse_routed_scaling_factor_in_topk
),
num_fused_shared_experts=self.num_fused_shared_experts,
fused_shared_experts_scaling_factor=fused_shared_experts_scaling_factor,
is_fp4_experts=(
quant_config is not None
@@ -1016,7 +1013,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
layer_id: int = 0,
prefix: str = "layer",
num_fused_shared_experts: int = 0,
is_nextn: bool = False,
alt_stream: Optional[torch.cuda.Stream] = None,
) -> None:
@@ -1094,7 +1090,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
quant_config=quant_config,
layer_id=self.layer_id,
prefix=prefix,
num_fused_shared_experts=num_fused_shared_experts,
alt_stream=alt_stream,
)
else:
@@ -1210,7 +1205,6 @@ class BailingMoELinearModel(nn.Module):
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
num_fused_shared_experts: int = 0,
) -> None:
super().__init__()
self.pp_group = get_parallel().pp_group
@@ -1258,7 +1252,6 @@ class BailingMoELinearModel(nn.Module):
decoder_kwargs = {
"quant_config": quant_config,
"layer_id": layer_idx,
"num_fused_shared_experts": num_fused_shared_experts,
}
return BailingMoELinearDecoderLayer(
layer_config,
@@ -1394,14 +1387,14 @@ class BailingMoeV3ForCausalLM(nn.Module):
self.config = config
self.quant_config = quant_config
# Determine num_fused_shared_experts
self.determine_num_fused_shared_experts()
# The weight loader reads this after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.log_shared_experts_fusion()
self.model = BailingMoELinearModel(
self.config,
quant_config,
prefix=add_prefix("model", prefix),
num_fused_shared_experts=self.num_fused_shared_experts,
)
if self.pp_group.is_last_rank:
@@ -1522,26 +1515,25 @@ class BailingMoeV3ForCausalLM(nn.Module):
)
return None
def determine_num_fused_shared_experts(self):
self.num_fused_shared_experts = (
0
if is_shared_experts_fusion_disabled()
else _get_bailing_num_shared_experts(self.config)
)
if self.num_fused_shared_experts == 0:
return
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
"""Shared experts fused as routed slots when fusion is on."""
num_shared_experts = _get_bailing_num_shared_experts(hf_config)
# Safety check: current CUDA implementation only supports num_fused_shared_experts == 1.
# The grouped_topk_gpu and _post_process_topk_ids functions only handle the last column,
# which is incorrect when num_fused_shared_experts > 1.
# AMD platform with aiter handles this correctly via fused_append_shared_experts kernel.
if self.num_fused_shared_experts > 1 and not _is_hip:
if num_shared_experts > 1 and not _is_hip:
raise ValueError(
f"num_fused_shared_experts > 1 ({self.num_fused_shared_experts}) is not "
f"num_fused_shared_experts > 1 ({num_shared_experts}) is not "
f"supported on CUDA platform. The current TopK implementation only handles "
f"one fused shared expert. AMD platform with aiter supports multiple shared experts."
)
return num_shared_experts
def log_shared_experts_fusion(self):
if self.num_fused_shared_experts == 0:
return
moe_ep_size = get_parallel().moe_ep_size
if moe_ep_size > 1:
log_info_on_rank0(
+2 -1
View File
@@ -33,6 +33,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
from sglang.srt.layers.quantization import Fp8Config
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.vocab_parallel_embedding import (
@@ -284,7 +285,7 @@ class DeepseekV3ForCausalLMNextN(DeepseekV3ForCausalLM):
self.quant_config = quant_config
# if not set, model load will be broken in DeepseekV3ForCausalLM load_weights()
self.pp_group = get_parallel().pp_group
self.determine_num_fused_shared_experts()
self.num_fused_shared_experts = get_num_fused_shared_experts()
nextn_quant_config = self._resolve_nextn_quant_config(config, quant_config)
self.model = DeepseekModelNextN(
+19 -7
View File
@@ -1421,21 +1421,33 @@ def build_qwen2_decoder_as_encoder(
return decoder_as_encoder
def _fused_moe_text_config(hf_config):
"""Return the nested fusable MoE config, or None for other branches."""
text_config = hf_config.text_config
if is_ocr2_config(hf_config) or not (
text_config.topk_method == "noaux_tc" or text_config.use_mla
):
return None
return text_config
class DeepseekOCRForCausalLM(nn.Module):
@staticmethod
def shared_experts_fusion_disable_reason(hf_config, quant_config):
text_config = hf_config.text_config
# Class-level hook: called before the model is built, so no `self.is_ocr2` yet.
if is_ocr2_config(hf_config) or not (
text_config.topk_method == "noaux_tc" or text_config.use_mla
):
# Those branches build the dense DeepseekForCausalLM, which has no
# shared experts to fuse.
text_config = _fused_moe_text_config(hf_config)
if text_config is None:
return None
return DeepseekV2ForCausalLM.shared_experts_fusion_disable_reason(
text_config, quant_config
)
@staticmethod
def num_fusable_shared_experts(hf_config) -> int:
text_config = _fused_moe_text_config(hf_config)
if text_config is None:
return 0
return DeepseekV2ForCausalLM.num_fusable_shared_experts(text_config)
def __init__(
self,
*,
+15 -15
View File
@@ -129,10 +129,10 @@ from sglang.srt.layers.moe.utils import (
RoutingMethodType,
adds_replicated_output_to_partial,
filter_moe_weight_param_global_expert,
get_num_fused_shared_experts,
has_per_rank_fused_shared_slots,
is_deepep_class_backend,
is_sbo_enabled,
is_shared_experts_fusion_disabled,
is_tbo_enabled,
should_add_replicated_moe_output,
sum_post_experts_output,
@@ -676,11 +676,13 @@ class DeepseekV2MoE(nn.Module):
n_shared_experts = (
0 if config.n_shared_experts is None else int(config.n_shared_experts)
)
_fusion_disabled = is_shared_experts_fusion_disabled()
# num_fused_shared_experts drives weight remapping in deepseek_weight_loader:
# mlp.shared_experts → mlp.experts.256 when > 0.
self.num_fused_shared_experts = 0 if _fusion_disabled else n_shared_experts
self.num_fused_shared_experts = get_num_fused_shared_experts()
assert self.num_fused_shared_experts in (0, n_shared_experts), (
f"The model fuses {self.num_fused_shared_experts} shared experts, but "
f"this MoE block has {n_shared_experts}."
)
# DeepEP and MegaMOE shared expert fusion: shared expert is fused into
# the same MoE kernel as a local expert at each EP rank. Expert layout
@@ -762,9 +764,8 @@ class DeepseekV2MoE(nn.Module):
if self.is_hash and not (is_nextn and is_deepseek_v4):
self.topk = HashTopK(
topk=config.num_experts_per_tok + self.num_fused_shared_experts,
topk=config.num_experts_per_tok,
num_experts=config.n_routed_experts,
num_fused_shared_experts=self.num_fused_shared_experts,
vocab_size=config.vocab_size,
scoring_func=config.scoring_func,
routed_scaling_factor=self.routed_scaling_factor,
@@ -778,12 +779,11 @@ class DeepseekV2MoE(nn.Module):
else:
# Default: grouped noaux_tc top-k. Covers V3/V3.2/GLM-5/Glm4MoeLite.
topk_kwargs = dict(
top_k=config.num_experts_per_tok + self.num_fused_shared_experts,
top_k=config.num_experts_per_tok,
layer_id=self.layer_id,
renormalize=config.norm_topk_prob,
use_grouped_topk=True,
num_expert_group=config.n_group,
num_fused_shared_experts=self.num_fused_shared_experts,
topk_group=config.topk_group,
scoring_func=config.scoring_func,
correction_bias=self.gate.e_score_correction_bias,
@@ -3282,7 +3282,8 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
self.pp_group = get_parallel().pp_group
self.config = config
self.quant_config = quant_config
self.determine_num_fused_shared_experts()
# The weight loader reads this after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.use_dsa = is_deepseek_dsa(config)
self.model = DeepseekV2Model(
config, quant_config, prefix=add_prefix("model", prefix)
@@ -3330,7 +3331,7 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
"""Why this checkpoint cannot fuse its shared expert, or None.
Evaluated by the loader once per runner, before any layer is built (see
``install_shared_experts_fusion_decision``), so it takes the config and
``resolve_shared_experts_config``), so it takes the config and
quantization it is asked about rather than reading an instance.
"""
# Need to disable if quant precision mismatch, even if
@@ -3397,11 +3398,10 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
return "Deepseek V3/R1 W4AFP8/W4A16 model uses different quant method for routed experts and shared experts."
return None
def determine_num_fused_shared_experts(self):
# The decision was installed by the loader; this only reads it.
self.num_fused_shared_experts = (
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
)
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
"""Return the model's fusable shared-expert count."""
return hf_config.n_shared_experts or 0
def get_input_embeddings(self) -> nn.Embedding:
return self.model.embed_tokens
+7 -7
View File
@@ -90,7 +90,7 @@ from sglang.srt.layers.logits_processor import LogitsMetadata, LogitsProcessor
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.utils import (
is_shared_experts_fusion_disabled,
get_num_fused_shared_experts,
should_skip_post_experts_all_reduce,
uses_per_rank_fused_shared_slots,
)
@@ -4447,7 +4447,8 @@ class DeepseekV4ForCausalLM(nn.Module):
self.tp_size = get_parallel().tp_size
self.quant_config = quant_config
self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
self.determine_num_fused_shared_experts()
# The weight loader reads this after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.vision = None
if (
config.model_type == "deepseek_v41"
@@ -4663,11 +4664,10 @@ class DeepseekV4ForCausalLM(nn.Module):
)
return None
def determine_num_fused_shared_experts(self):
# The decision was installed by the loader; this only reads it.
self.num_fused_shared_experts = (
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
)
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
"""Shared experts fused as routed slots when fusion is on."""
return hf_config.n_shared_experts or 0
def prepare_language_model_inputs(
self,
@@ -24,7 +24,7 @@ from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.vocab_parallel_embedding import (
@@ -702,6 +702,10 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
hf_config, quant_config
)
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
return DeepseekV4ForCausalLM.num_fusable_shared_experts(hf_config)
def __init__(
self,
config: DeepSeekV4Config,
@@ -711,9 +715,7 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
super().__init__()
self.config = config
self.quant_config = quant_config
self.num_fused_shared_experts = (
0 if is_shared_experts_fusion_disabled() else config.n_shared_experts
)
self.num_fused_shared_experts = get_num_fused_shared_experts()
dspark_config = parse_dspark_draft_config(draft_hf_config=config)
if not dspark_config.require_markov():
@@ -24,7 +24,10 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
from sglang.srt.layers.moe.utils import (
get_moe_a2a_backend,
get_num_fused_shared_experts,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.modelslim.modelslim import ModelSlimConfig
from sglang.srt.layers.vocab_parallel_embedding import (
@@ -264,7 +267,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM):
self.pp_group = get_parallel().pp_group
self.quant_config = quant_config
self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
self.determine_num_fused_shared_experts()
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.model = DeepseekV4ModelNextN(
config, quant_config, prefix=add_prefix("model", prefix)
+16 -3
View File
@@ -156,17 +156,30 @@ class DeepseekVL2MlpProjector(nn.Module):
return x
def _fused_moe_text_config(hf_config):
"""Return the nested fusable MoE config, or None for other branches."""
language_config = hf_config.language_config
return language_config if language_config.use_mla else None
class DeepseekVL2ForCausalLM(nn.Module):
@staticmethod
def shared_experts_fusion_disable_reason(hf_config, quant_config):
language_config = hf_config.language_config
if not language_config.use_mla:
text_config = _fused_moe_text_config(hf_config)
if text_config is None:
return None
# The language model is built without a quantization config.
return DeepseekV2ForCausalLM.shared_experts_fusion_disable_reason(
language_config, None
text_config, None
)
@staticmethod
def num_fusable_shared_experts(hf_config) -> int:
text_config = _fused_moe_text_config(hf_config)
if text_config is None:
return 0
return DeepseekV2ForCausalLM.num_fusable_shared_experts(text_config)
def __init__(
self,
config: DeepseekVL2Config,
@@ -80,7 +80,7 @@ from sglang.srt.layers.moe import (
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8_utils import (
block_quant_dequant,
@@ -278,9 +278,7 @@ class Dots3MoE(nn.Module):
super().__init__()
self.tp_size = get_parallel().tp_size
self.routed_scaling_factor = config.routed_scaling_factor
self.num_fused_shared_experts = (
0 if is_shared_experts_fusion_disabled() else config.n_shared_experts
)
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.config = config
self.layer_id = layer_id
self.alt_stream = alt_stream
@@ -314,12 +312,11 @@ class Dots3MoE(nn.Module):
)
self.topk = TopK(
top_k=config.num_experts_per_tok + self.num_fused_shared_experts,
top_k=config.num_experts_per_tok,
layer_id=self.layer_id,
renormalize=config.norm_topk_prob,
use_grouped_topk=True,
num_expert_group=config.n_group,
num_fused_shared_experts=self.num_fused_shared_experts,
topk_group=config.topk_group,
correction_bias=self.gate.e_score_correction_bias,
routed_scaling_factor=self.routed_scaling_factor,
@@ -1605,7 +1602,8 @@ class Dots3LanguageModelForCausalLM(nn.Module):
self.pp_group = get_parallel().pp_group
self.config = config
self.quant_config = quant_config
self.determine_num_fused_shared_experts()
# The weight loader reads this after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.model = Dots3Model(
config, quant_config, prefix=add_prefix("model", prefix)
)
@@ -1649,10 +1647,10 @@ class Dots3LanguageModelForCausalLM(nn.Module):
)
return None
def determine_num_fused_shared_experts(self):
self.num_fused_shared_experts = (
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
)
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
"""Shared experts fused as routed slots when fusion is on."""
return hf_config.n_shared_experts or 0
def get_input_embeddings(self) -> nn.Embedding:
return self.model.embed_tokens
@@ -2478,6 +2476,10 @@ class DotsNoteOmniForConditionalGeneration(nn.Module):
hf_config, quant_config
)
@staticmethod
def num_fusable_shared_experts(hf_config) -> int:
return Dots3LanguageModelForCausalLM.num_fusable_shared_experts(hf_config)
def __init__(
self,
config: PretrainedConfig,
@@ -14,6 +14,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -158,7 +159,7 @@ class Dots3NoteForCausalLMNextN(Dots3LanguageModelForCausalLM):
"g_proj",
]
}
self.determine_num_fused_shared_experts()
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.model = Dot3NoteModelNextN(
config, quant_config, prefix=add_prefix("model", prefix)
+15 -2
View File
@@ -38,17 +38,30 @@ from sglang.srt.runtime_context import get_parallel
from .dots_vlm_vit import DotsVisionTransformer
def _fused_moe_text_config(hf_config):
"""Return the nested fusable MoE config, or None for other branches."""
return None if hf_config.encoder_only else hf_config.language_config
class DotsVLMForCausalLM(nn.Module):
"""DotsVLM model for sglang inference"""
@staticmethod
def shared_experts_fusion_disable_reason(hf_config, quant_config):
if hf_config.encoder_only:
text_config = _fused_moe_text_config(hf_config)
if text_config is None:
return None
return DeepseekV2ForCausalLM.shared_experts_fusion_disable_reason(
hf_config.language_config, quant_config
text_config, quant_config
)
@staticmethod
def num_fusable_shared_experts(hf_config) -> int:
text_config = _fused_moe_text_config(hf_config)
if text_config is None:
return 0
return DeepseekV2ForCausalLM.num_fusable_shared_experts(text_config)
def __init__(
self, config: DotsVLMConfig, quant_config: Optional[QuantizationConfig] = None
) -> None:
+9 -13
View File
@@ -69,7 +69,7 @@ from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_t
from sglang.srt.layers.moe.utils import (
RoutingMethodType,
filter_moe_weight_param_global_expert,
is_shared_experts_fusion_disabled,
get_num_fused_shared_experts,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
@@ -414,9 +414,7 @@ class Glm4MoeSparseMoeBlock(nn.Module):
self.tp_size = get_parallel().tp_size
self.routed_scaling_factor = config.routed_scaling_factor
self.n_shared_experts = config.n_shared_experts
self.num_fused_shared_experts = (
0 if is_shared_experts_fusion_disabled() else config.n_shared_experts
)
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.config = config
self.layer_id = layer_id
@@ -450,7 +448,7 @@ class Glm4MoeSparseMoeBlock(nn.Module):
)
self.topk = TopK(
top_k=self.top_k + self.num_fused_shared_experts,
top_k=self.top_k,
layer_id=self.layer_id,
renormalize=config.norm_topk_prob,
use_grouped_topk=True,
@@ -458,7 +456,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
topk_group=config.topk_group,
correction_bias=self.gate.e_score_correction_bias,
routed_scaling_factor=self.routed_scaling_factor,
num_fused_shared_experts=self.num_fused_shared_experts,
apply_routed_scaling_factor_on_output=getattr(
self.experts, "should_fuse_routed_scaling_factor_in_topk", False
),
@@ -995,8 +992,8 @@ class Glm4MoeForCausalLM(nn.Module):
self.pp_group = get_parallel().pp_group
self.config = config
self.quant_config = quant_config
self.num_fused_shared_experts = 0
self.determine_num_fused_shared_experts()
# The weight loader reads this after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.model = Glm4MoeModel(
config, quant_config, prefix=add_prefix("model", prefix)
)
@@ -1033,11 +1030,10 @@ class Glm4MoeForCausalLM(nn.Module):
return "GLM-4.5 W4AFP8 model uses different quant method for routed experts and shared experts."
return None
def determine_num_fused_shared_experts(self):
# The decision was installed by the loader; this only reads it.
self.num_fused_shared_experts = (
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
)
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
"""Shared experts fused as routed slots when fusion is on."""
return hf_config.n_shared_experts or 0
def get_input_embeddings(self) -> nn.Embedding:
return self.model.embed_tokens
+9 -12
View File
@@ -66,7 +66,7 @@ from sglang.srt.layers.moe.topk import (
)
from sglang.srt.layers.moe.utils import (
filter_moe_weight_param_global_expert,
is_shared_experts_fusion_disabled,
get_num_fused_shared_experts,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.utils import PPMissingLayer
@@ -185,9 +185,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
self.tp_size = get_parallel().tp_size
self.routed_scaling_factor = config.routed_scaling_factor
self.n_shared_experts = config.n_shared_experts
self.num_fused_shared_experts = (
0 if is_shared_experts_fusion_disabled() else config.n_shared_experts
)
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.config = config
self.layer_id = layer_id
self.alt_stream = alt_stream
@@ -224,12 +222,11 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
)
self.topk = TopK(
top_k=config.num_experts_per_tok + self.num_fused_shared_experts,
top_k=config.num_experts_per_tok,
layer_id=self.layer_id,
renormalize=config.norm_topk_prob,
use_grouped_topk=True,
num_expert_group=config.n_group,
num_fused_shared_experts=self.num_fused_shared_experts,
topk_group=config.topk_group,
correction_bias=self.gate.e_score_correction_bias,
quant_config=quant_config,
@@ -704,7 +701,8 @@ class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
self.config = config
self.quant_config = quant_config
self.pp_group = get_parallel().pp_group
self.determine_num_fused_shared_experts()
# The weight loader reads this after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.model = Glm4MoeLiteModel(
config, quant_config, prefix=add_prefix("model", prefix)
)
@@ -749,11 +747,10 @@ class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
return "GLM-4.5 or GLM-4.6 cannot use shared experts fusion optimization under expert parallelism."
return None
def determine_num_fused_shared_experts(self):
# The decision was installed by the loader; this only reads it.
self.num_fused_shared_experts = (
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
)
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
"""Shared experts fused as routed slots when fusion is on."""
return hf_config.n_shared_experts or 0
def get_input_embeddings(self) -> nn.Embedding:
return self.model.embed_tokens
@@ -27,6 +27,7 @@ from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -148,10 +149,8 @@ class Glm4MoeLiteForCausalLMNextN(Glm4MoeLiteForCausalLM):
quant_config = None
self.quant_config = quant_config
# The draft's own gate (its quantization can differ from the
# target's); the decoder below reads the ACTIVE decision as it builds,
# and num_fused_shared_experts drives the inherited loader's remap.
self.determine_num_fused_shared_experts()
# Drives the inherited loader's shared-expert remap after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.model = Glm4MoeLiteModelNextN(
config, quant_config, prefix=add_prefix("model", prefix)
+3 -6
View File
@@ -27,6 +27,7 @@ from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -129,12 +130,8 @@ class Glm4MoeForCausalLMNextN(Glm4MoeForCausalLM):
quant_config = None
self.quant_config = quant_config
# The draft's own gate: its quantization can differ from the
# target's, and the decoder below reads the ACTIVE decision while it
# builds. Also sets num_fused_shared_experts, which drives the
# inherited loader's shared-expert remap.
self.num_fused_shared_experts = 0
self.determine_num_fused_shared_experts()
# Drives the inherited loader's shared-expert remap after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
self.model = Glm4MoeModelNextN(
config, quant_config, prefix=add_prefix("model", prefix)
+10 -10
View File
@@ -10,7 +10,7 @@ from sglang.srt.layers.attention import vision_utils
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
from sglang.srt.layers.pooler import Pooler, PoolingType
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.utils import PPMissingLayer
@@ -44,8 +44,10 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration):
self.use_data_parallel = get_mm().mm_enable_dp_encoder
vision_utils.update_vit_attn_dummy_heads_config(self.config)
self.quant_config = quant_config
self.num_fused_shared_experts = 0
self.determine_num_fused_shared_experts()
# The weight loader reads this after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
if self.num_fused_shared_experts:
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
self.model = Glm4MoeModel(
config,
@@ -97,15 +99,13 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration):
return "Shared experts fusion is not supported when Deepep MoE backend is enabled."
return None
def determine_num_fused_shared_experts(self):
# The decision was installed by the loader; this only reads it.
if is_shared_experts_fusion_disabled():
return
self.num_fused_shared_experts = self.config.n_shared_experts
assert self.num_fused_shared_experts == 1, (
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
"""Shared experts fused as routed slots when fusion is on."""
assert hf_config.n_shared_experts == 1, (
"Only 1 fused shared expert is supported for Glm4vMoeForConditionalGeneration"
)
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
return hf_config.n_shared_experts
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]], is_nextn=False):
if is_nextn:
+12 -11
View File
@@ -54,7 +54,7 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.utils import (
get_moe_a2a_backend,
get_moe_runner_backend,
is_shared_experts_fusion_disabled,
get_num_fused_shared_experts,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.utils import is_layer_skipped
@@ -1277,7 +1277,10 @@ class Glm5NextForConditionalGeneration(nn.Module):
self.lm_head = None
self.logits_processor = None
if not self.encoder_only:
self.determine_num_fused_shared_experts()
# The weight loader reads this after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
if self.num_fused_shared_experts:
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
self.model = Glm5NextModel(
text_config, quant_config, prefix=add_prefix("model", prefix)
)
@@ -1491,16 +1494,14 @@ class Glm5NextForConditionalGeneration(nn.Module):
return reason
return None
def determine_num_fused_shared_experts(self):
self.num_fused_shared_experts = (
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
"""Return the model's fusable shared-expert count."""
num_shared_experts = hf_config.text_config.n_shared_experts
assert num_shared_experts == 1, (
f"Only 1 fused shared expert is supported for {cls.__name__}"
)
if self.num_fused_shared_experts == 0:
return
assert self.num_fused_shared_experts == 1, (
f"Only 1 fused shared expert is supported for {type(self).__name__}"
)
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
return num_shared_experts
def wants_prefill_autotune(self) -> bool:
backend = get_moe_runner_backend()
+15 -2
View File
@@ -632,6 +632,11 @@ def mm_projection_auto(
return projected.reshape(-1, projected.shape[-1])
def _fused_moe_text_config(hf_config):
"""Return the nested fusable MoE config, or None for other branches."""
return None if hf_config.encoder_only else hf_config.text_config
class KimiK25ForConditionalGeneration(nn.Module):
supports_cuda_vmm_feature_transport = True
@@ -646,12 +651,20 @@ class KimiK25ForConditionalGeneration(nn.Module):
@staticmethod
def shared_experts_fusion_disable_reason(hf_config, quant_config):
if hf_config.encoder_only:
text_config = _fused_moe_text_config(hf_config)
if text_config is None:
return None
return DeepseekV3ForCausalLM.shared_experts_fusion_disable_reason(
hf_config.text_config, quant_config
text_config, quant_config
)
@staticmethod
def num_fusable_shared_experts(hf_config) -> int:
text_config = _fused_moe_text_config(hf_config)
if text_config is None:
return 0
return DeepseekV3ForCausalLM.num_fusable_shared_experts(text_config)
def __init__(
self,
config: KimiK25Config,
+15 -2
View File
@@ -120,15 +120,28 @@ def _language_model_config(config: KimiVLConfig):
return text_config
def _fused_moe_text_config(hf_config):
"""Return the nested fusable MoE config, or None for other branches."""
return None if hf_config.encoder_only else _language_model_config(hf_config)
class KimiVLForConditionalGeneration(nn.Module):
@staticmethod
def shared_experts_fusion_disable_reason(hf_config, quant_config):
if hf_config.encoder_only:
text_config = _fused_moe_text_config(hf_config)
if text_config is None:
return None
return DeepseekV2ForCausalLM.shared_experts_fusion_disable_reason(
_language_model_config(hf_config), quant_config
text_config, quant_config
)
@staticmethod
def num_fusable_shared_experts(hf_config) -> int:
text_config = _fused_moe_text_config(hf_config)
if text_config is None:
return 0
return DeepseekV2ForCausalLM.num_fusable_shared_experts(text_config)
def __init__(
self,
config: KimiVLConfig,
+12 -15
View File
@@ -67,7 +67,7 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import (
get_moe_a2a_backend,
is_shared_experts_fusion_disabled,
get_num_fused_shared_experts,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.mxfp8_input import (
@@ -384,9 +384,7 @@ class MiniMaxM3MoE(nn.Module):
self.tp_size = get_parallel().tp_size
self.alt_stream = alt_stream
self.n_shared_experts = getattr(config, "n_shared_experts", None)
self.num_fused_shared_experts = (
0 if is_shared_experts_fusion_disabled() else config.n_shared_experts
)
self.num_fused_shared_experts = get_num_fused_shared_experts()
if self.tp_size > config.num_local_experts:
raise ValueError(
@@ -424,12 +422,11 @@ class MiniMaxM3MoE(nn.Module):
gate_up_interleaved=False,
)
self.topk = TopK(
top_k=config.num_experts_per_tok + self.num_fused_shared_experts,
top_k=config.num_experts_per_tok,
renormalize=True,
layer_id=layer_id,
scoring_func=config.scoring_func,
correction_bias=self.e_score_correction_bias,
num_fused_shared_experts=self.num_fused_shared_experts,
routed_scaling_factor=self.routed_scaling_factor,
apply_routed_scaling_factor_on_output=True,
)
@@ -1763,8 +1760,10 @@ class MiniMaxM3SparseForCausalLM(nn.Module):
self.quant_config = quant_config
self.pp_group = get_parallel().pp_group
self.num_fused_shared_experts = 0
self.determine_num_fused_shared_experts()
# The weight loader reads this after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
if self.num_fused_shared_experts:
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
self.model = MiniMaxM3Model(
config, quant_config, prefix=add_prefix("model", prefix)
@@ -1811,15 +1810,13 @@ class MiniMaxM3SparseForCausalLM(nn.Module):
return "Shared experts fusion is not supported when Deepep MoE backend is enabled."
return None
def determine_num_fused_shared_experts(self):
# The decision was installed by the loader; this only reads it.
if is_shared_experts_fusion_disabled():
return
self.num_fused_shared_experts = self.config.n_shared_experts
assert self.num_fused_shared_experts == 1, (
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
"""Shared experts fused as routed slots when fusion is on."""
assert hf_config.n_shared_experts == 1, (
"Only 1 fused shared expert is supported for MiniMax-M3"
)
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
return hf_config.n_shared_experts
def set_eagle3_layers_to_capture(self, layer_ids: Optional[list[int]] = None):
if not self.pp_group.is_last_rank:
+11 -12
View File
@@ -10,7 +10,7 @@ import torch.nn as nn
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.utils import (
get_moe_a2a_backend,
is_shared_experts_fusion_disabled,
get_num_fused_shared_experts,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.utils import PPMissingLayer
@@ -88,8 +88,10 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module):
self.use_data_parallel = get_mm().mm_enable_dp_encoder
self.num_fused_shared_experts = 0
self._determine_num_fused_shared_experts()
# The weight loader reads this after the build.
self.num_fused_shared_experts = get_num_fused_shared_experts()
if self.num_fused_shared_experts:
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
vision_config_raw = config.vision_config
assert vision_config_raw is not None, "vision_config is required"
@@ -177,15 +179,12 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module):
)
return None
def _determine_num_fused_shared_experts(self) -> None:
# The decision was installed by the loader; this only reads it.
if is_shared_experts_fusion_disabled():
return
self.num_fused_shared_experts = self.config.text_config.n_shared_experts
assert self.num_fused_shared_experts == 1, (
"Only 1 fused shared expert is supported"
)
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
@classmethod
def num_fusable_shared_experts(cls, hf_config) -> int:
"""Return the model's fusable shared-expert count."""
num_shared_experts = hf_config.text_config.n_shared_experts
assert num_shared_experts == 1, "Only 1 fused shared expert is supported"
return num_shared_experts
@classmethod
def get_model_config_for_expert_location(cls, config):
+17 -3
View File
@@ -68,19 +68,33 @@ class VisionEncoderArgs:
mm_projector_id: str = ""
def _fused_moe_text_config(hf_config):
"""Return the nested fusable MoE config, or None for other branches."""
text_config = hf_config.text_config
if getattr(text_config, "model_type", "") != "deepseek_v3":
return None
return text_config
class PixtralForConditionalGeneration(nn.Module):
merge_by_field_config = True
@staticmethod
def shared_experts_fusion_disable_reason(hf_config, quant_config):
text_config = hf_config.text_config
if getattr(text_config, "model_type", "") != "deepseek_v3":
# The GQA text config builds the dense Mistral backbone.
text_config = _fused_moe_text_config(hf_config)
if text_config is None:
return None
return MistralLarge3ForCausalLM.shared_experts_fusion_disable_reason(
text_config, quant_config
)
@staticmethod
def num_fusable_shared_experts(hf_config) -> int:
text_config = _fused_moe_text_config(hf_config)
if text_config is None:
return 0
return MistralLarge3ForCausalLM.num_fusable_shared_experts(text_config)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
+1 -13
View File
@@ -557,19 +557,7 @@ class MoeFlags(_FlagGroupBase):
tbo_token_distribution_threshold: float | None = None
disable_fp4_allgather: bool | None = None
quantization: str | None = None
# The shared-experts-fusion decision, per runner — the runner_backend /
# speculative_runner_backend shape. Both leaves are seeded from the config
# intent by ``initialize_moe_config``; each MoE model's gate
# (determine_num_fused_shared_experts) refines the ACTIVE leaf, both ways,
# before its layers build and read it. ``speculative_moe_backend_context``
# brackets a draft's build: on exit the draft's effective decision is
# persisted onto the speculative leaf (inspectable afterwards) and the
# target's ACTIVE value returns.
disable_shared_experts_fusion: bool | None = None
speculative_disable_shared_experts_fusion: bool | None = None
# Lifecycle marker (the capture.disable_dispose_tensor family): set while
# speculative_moe_backend_context is active, so a draft gate's write also
# lands on the speculative leaf.
# True while draft construction/backend scopes are active.
in_speculative_scope: bool = False
# Draft construction/execution uses a separate one-sided A2A workspace from
# the target model's concurrently live CUDA graphs.
@@ -300,7 +300,8 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
self._metadata_glue = None
def _replay_graph(self, shape_key, forward_batch):
return self.backend.replay(shape_key, forward_batch)
with self.model_context():
return self.backend.replay(shape_key, forward_batch)
# -----------------------------------------------------------------
# Helpers
@@ -307,7 +307,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
)
def _replay_graph(self, shape_key, forward_batch):
return self.backend.replay(shape_key, forward_batch)
with self.model_context():
return self.backend.replay(shape_key, forward_batch)
def _cache_loc_dtype(self):
return torch.int64
@@ -203,7 +203,8 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
return ShapeKey(size=bs)
def _replay_graph(self, shape_key, forward_batch):
return self.backend.replay(shape_key, forward_batch)
with self.model_context():
return self.backend.replay(shape_key, forward_batch)
def can_run_graph(self, forward_batch: ForwardBatch):
# Uniform-width replay invariant: the batch's actual per-request width
@@ -227,7 +227,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
)
def _replay_graph(self, shape_key, forward_batch):
return self.backend.replay(shape_key, forward_batch)
with self.model_context():
return self.backend.replay(shape_key, forward_batch)
def _make_graph_key(self, bs, stream_idx=None, variant_label=None):
return ShapeKey(size=bs)
@@ -412,11 +413,12 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
output_cache_loc_backup = forward_batch.out_cache_loc
hidden_states_backup = forward_batch.spec_info.hidden_states
ret = self.model_runner.model.forward(
forward_batch.input_ids,
forward_batch.positions,
forward_batch,
)
with self.model_context():
ret = self.model_runner.model.forward(
forward_batch.input_ids,
forward_batch.positions,
forward_batch,
)
if self.eagle_worker.chain_mtp_hidden_states and ret.hidden_states is not None:
buffers.hidden_states[:num_tokens].copy_(ret.hidden_states[:num_tokens])
@@ -1083,7 +1085,8 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
with device_timer_ctx(
first.model_runner.device_timer, "eagle_draft_extend"
):
outs = first.backend.replay(shape_key, self._replay_spec_info)
with first.model_context():
outs = first.backend.replay(shape_key, self._replay_spec_info)
raw_bs = self.raw_bs
self._cached = {}
non_null = [r for r in self.runners if r is not None]
@@ -49,9 +49,9 @@ class RoutedExpertsCapturer(BaseTopkCapturer):
) -> Optional["RoutedExpertsCapturer"]:
if not get_exec().features.enable_return_routed_experts:
return None
# The model's own attribute is the baked decision (0 when its gate
# disabled fusion); the ACTIVE flag can be holding another runner's.
num_fused_shared_experts = getattr(model, "num_fused_shared_experts", 0)
# The loader attaches this to every entry model, including wrappers.
# Size from the owner, independent of which runner is currently active.
num_fused_shared_experts = model.shared_experts_config.num_fused_shared_experts
return RoutedExpertsCapturer(
model_config,
num_tokens=num_tokens,
@@ -8,6 +8,7 @@ from unittest.mock import patch
import torch
from torch import nn
from sglang.srt.configs.shared_experts import SharedExpertsConfig
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
@@ -672,6 +673,7 @@ class _EagleDraftExtendForward(nn.Module):
device: str,
):
super().__init__()
self.shared_experts_config = SharedExpertsConfig()
self.module = module
self.token_embed = nn.Embedding(
vocab_size, hidden_size, dtype=dtype, device=device
@@ -7,6 +7,7 @@ from unittest.mock import patch
import torch
from torch import nn
from sglang.srt.configs.shared_experts import SharedExpertsConfig
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseConfig
from sglang.srt.model_executor.forward_batch_info import (
@@ -153,6 +154,10 @@ class _DummyTpGroup:
class _TinyDraftModel(nn.Module):
def __init__(self):
super().__init__()
self.shared_experts_config = SharedExpertsConfig()
def forward(self, *args, **kwargs):
raise RuntimeError("EAGLEDraftCudaGraphRunner should call worker.draft_forward")