diff --git a/.agents/skills/sglang-runtime-context/SKILL.md b/.agents/skills/sglang-runtime-context/SKILL.md index ea02a3086c..b441c292b3 100644 --- a/.agents/skills/sglang-runtime-context/SKILL.md +++ b/.agents/skills/sglang-runtime-context/SKILL.md @@ -135,16 +135,23 @@ bag to override at all. instance: **constructor arguments** (`ModelRunner(draft_attention_backend=...)`, `MMEncoder(gpu_id=...)`) and **runner attributes holding the resolved value** (`model_runner.kv_cache_dtype_str`, `prefill_attention_backend_str`, - `num_fused_shared_experts`, `linear_attn_backends`) — threaded to consumers as + `linear_attn_backends`) — threaded to consumers as arguments, never backfilled onto a shared object. A per-runner choice also stays *out* of the bags: recording it there is how a second runner inherits the first - one's answer, which is exactly the bug `linear_attn_backends` replaced. The one sanctioned bend in that rule is - *scoped*: `ModelRunner._load_format_scope` exposes the draft's - `--speculative-draft-load-format` through `get_model().override(load_format=...)` - for exactly the duration of the draft build, because model construction - reads that bag leaf — the override restores on exit, so nothing outlives - the scope. When there is a runner in hand, read its - stamp; that is a different rule from "read the instance". + one's answer, which is exactly the bug `linear_attn_backends` replaced. The two + sanctioned bends in that rule are *scoped*. `ModelRunner._load_format_scope` + exposes the draft's `--speculative-draft-load-format` through + `get_model().override(load_format=...)` for exactly the duration of the draft + build, because model construction reads that bag leaf — the override restores + on exit, so nothing outlives the scope. The shared-expert layout is another scoped view: + `get_model().shared_experts` selects the immutable `SharedExpertsConfig` owned + by the loaded model (`model.shared_experts_config`). The loader binds it for + construction, and runners bind it for execution, warmup and graph capture. + Both the fused count and ordinary TopK slot count belong to this object; + neither is a separately mutable flag. A scope selects a built layout and + restores the previous reference on exit, including exceptions. It cannot + resize weights, buffers or a captured graph. When the model is in hand, read + its owned config directly (as the routed-experts capturer does). - **Per-instance boundaries** — the tokenizer-manager family, everything under `entrypoints/`, and the tokenizer-process multimodal processors read the bags. The old justification for keeping them on `self.server_args` ("several @@ -327,7 +334,8 @@ where an object was handed one; it is not a global accessor. dist init to answer. - **this runner's resolved value** → the runner (`prefill_attention_backend_str`, `kv_cache_dtype_str`, - `draft_attention_backend`, `num_fused_shared_experts` on the model). + `draft_attention_backend`); for what the model's layers bake in at + construction, the layer (`num_fused_shared_experts` on a MoE block). `self.server_args.field` is still right for handed per-instance config (see "Reads that legitimately stay on a ServerArgs instance" above for the full set — @@ -451,13 +459,18 @@ Never assign `server_args` fields from model code. Declare instead its checkpoint drives (shared-experts fusion) is a question the *loader* asks the model class — `shared_experts_fusion_disable_reason(hf_config, quant_config)`, a classmethod answering without an instance — at the single - model-instantiation point, and - `install_shared_experts_fusion_decision` writes the answer to the ACTIVE moe - flag before that model's layers build and read it - (`is_shared_experts_fusion_disabled`, config-intent fallback). - `draft_model_build_scope` brackets every draft build and routes the draft's - answer to the speculative leaf, so a draft's decision never overwrites the - target's. A process-level load-time fact (the sm80 dtype fallback — + model-instantiation point, together with how many shared experts it fuses + (`num_fusable_shared_experts(hf_config)`; an entry class without both hooks + fuses none). `resolve_shared_experts_config` returns one immutable config; + the loader scopes it through `get_model().override(shared_experts=config)` + while building and retains it on the resulting model. `num_shared_topk_slots` + derives from the fused count and frozen Waterfill policy, so ordinary routing + and Waterfill cannot both claim the shared slots. `TopK`/`HashTopK` retain the + same config; `top_k` counts only routed experts at construction. + `draft_model_build_scope` still selects draft boundary/MMA preferences; it + does not save shared-expert flags. Target and draft each own their layout. + These values have model lifetime, so they do not belong in `get_forward()`. + A process-level load-time fact (the sm80 dtype fallback — device-driven, identical for every runner) records directly via `get_context().override`. diff --git a/python/sglang/srt/arg_groups/fields/model.py b/python/sglang/srt/arg_groups/fields/model.py index c7ba75bc36..65ede5cd25 100644 --- a/python/sglang/srt/arg_groups/fields/model.py +++ b/python/sglang/srt/arg_groups/fields/model.py @@ -39,6 +39,12 @@ class Model(msgspec.Struct): doc="Whether weight loading overlaps startup.", ) + # Runtime-only binding: loaders/runners scope the immutable model-owned value. + shared_experts = Derived( + fn="sglang.srt.configs.shared_experts.default_shared_experts_config", + doc="Shared-expert layout of the model being constructed or executed.", + ) + # ------------------------------------------------------------------------- # Model and tokenizer # ------------------------------------------------------------------------- diff --git a/python/sglang/srt/configs/shared_experts.py b/python/sglang/srt/configs/shared_experts.py new file mode 100644 index 0000000000..73e2fd5f1b --- /dev/null +++ b/python/sglang/srt/configs/shared_experts.py @@ -0,0 +1,21 @@ +"""Resolved shared-expert layout for one loaded model.""" + +import msgspec + + +class SharedExpertsConfig(msgspec.Struct, frozen=True): + """Immutable layout shared by a model, its routers and its captured graphs.""" + + fusion_disabled: bool = False + num_fused_shared_experts: int = 0 + use_waterfill: bool = False + + @property + def num_shared_topk_slots(self) -> int: + """Shared id/weight slots emitted before Waterfill expansion.""" + return 0 if self.use_waterfill else self.num_fused_shared_experts + + +def default_shared_experts_config(args) -> SharedExpertsConfig: + """Publication has no loaded model yet; retain only the startup intent.""" + return SharedExpertsConfig(fusion_disabled=args.disable_shared_experts_fusion) diff --git a/python/sglang/srt/layers/moe/hash_topk.py b/python/sglang/srt/layers/moe/hash_topk.py index 852cf223c5..205ae1ab9f 100644 --- a/python/sglang/srt/layers/moe/hash_topk.py +++ b/python/sglang/srt/layers/moe/hash_topk.py @@ -24,9 +24,8 @@ from sglang.srt.layers.moe.topk import ( ) from sglang.srt.layers.moe.utils import ( has_per_rank_fused_shared_slots, - shared_topk_slots, - waterfill_routes_shared_experts, ) +from sglang.srt.runtime_context import get_model from sglang.srt.utils import is_hip, is_npu, is_xpu logger = logging.getLogger(__name__) @@ -41,7 +40,7 @@ class HashTopK(nn.Module): self, topk, num_experts, - num_fused_shared_experts, + *, vocab_size, scoring_func="sqrtsoftplus", routed_scaling_factor=1.5, @@ -51,20 +50,18 @@ class HashTopK(nn.Module): super().__init__() self.layer_id = layer_id - self.enable_waterfill = waterfill_routes_shared_experts( - num_fused_shared_experts - ) + self.shared_experts = get_model().shared_experts + self.enable_waterfill = self.shared_experts.use_waterfill self.waterfill_balancer = None - num_routed_experts_per_token = topk - num_fused_shared_experts self.num_experts = num_experts - # Trailing columns the hash router fills with fused shared experts. - self.num_shared_topk_slots = shared_topk_slots(num_fused_shared_experts) - self.topk = num_routed_experts_per_token + self.num_shared_topk_slots + # topk counts routed experts; the trailing columns the hash router fills + # with fused shared experts follow them. + self.topk = topk + self.num_shared_topk_slots self.routed_scaling_factor = routed_scaling_factor self.score_func = scoring_func self.tid2eid = nn.Parameter( - torch.empty(vocab_size, num_routed_experts_per_token, dtype=torch.int32), + torch.empty(vocab_size, topk, dtype=torch.int32), requires_grad=False, ) self._init_default_tid2eid() @@ -78,6 +75,10 @@ class HashTopK(nn.Module): "with fused shared experts; pass --disable-shared-experts-fusion." ) + @property + def num_shared_topk_slots(self) -> int: + return self.shared_experts.num_shared_topk_slots + def _init_default_tid2eid(self) -> None: topk = self.tid2eid.shape[1] if topk == 0: @@ -286,7 +287,7 @@ class HashTopK(nn.Module): num_physical_routed_experts, TopKConfig( top_k=self.topk, - num_shared_topk_slots=num_shared_topk_slots, + shared_experts=self.shared_experts, routed_scaling_factor=self.routed_scaling_factor, apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output, ), diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index d10fe60434..52b5754bc5 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -38,10 +38,12 @@ import triton.language as tl if TYPE_CHECKING: from triton_kernels.tensor_details.ragged_tensor import RaggedTensorMetadata +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.runtime_context import ( get_exec, get_forward, get_lora, + get_model, get_parallel, get_server_args, ) @@ -106,8 +108,6 @@ from sglang.srt.layers.moe import ( ) from sglang.srt.layers.moe.utils import ( has_per_rank_fused_shared_slots, - shared_topk_slots, - waterfill_routes_shared_experts, ) from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer from sglang.srt.utils import ( @@ -258,9 +258,7 @@ class TopKConfig: topk_group: Optional[int] = None num_expert_group: Optional[int] = None renormalize: bool = True - # Trailing columns of top_k that the router fills with fused shared experts. - # Zero when Waterfill appends the shared expert after routing. - num_shared_topk_slots: int = 0 + shared_experts: SharedExpertsConfig = SharedExpertsConfig() custom_routing_function: Optional[Callable] = None correction_bias: Optional[torch.Tensor] = None torch_native: bool = False @@ -285,6 +283,10 @@ class TopKConfig: default=None, init=False, repr=False, compare=False ) + @property + def num_shared_topk_slots(self) -> int: + return self.shared_experts.num_shared_topk_slots + def correction_bias_for_dtype(self, dtype: torch.dtype) -> Optional[torch.Tensor]: """Return correction bias in ``dtype``, reusing a per-TopK lazy copy.""" correction_bias = self.correction_bias @@ -588,12 +590,11 @@ def _simulate_balanced_routing( class TopK(BaseFusedOp): """ Parameters: - --top_k: The all number of top experts selected per token, including the fused shared expert(s). + --top_k: the number of routed experts selected per token. When the model being built + fuses shared experts (``get_num_fused_shared_experts``), their columns follow: written + by the router (``TopKConfig.num_shared_topk_slots``), or appended by Waterfill. --layer_id: global index of the owning layer. Required: routed-experts capture and per-layer EPLB state are keyed by it. - --num_fused_shared_experts: shared experts the experts layer runs as extra expert slots. - Their trailing top-k columns are written by the router, or by Waterfill when it is - enabled (``TopKConfig.num_shared_topk_slots`` then excludes them). --routed_scaling_factor: the scaling factor for routed experts in topk_weights. --fused_shared_experts_scaling_factor: absolute appended weight on the global AITER path; otherwise multiplies the router's existing shared weight under standard EP. @@ -609,7 +610,6 @@ class TopK(BaseFusedOp): topk_group: Optional[int] = None, num_expert_group: Optional[int] = None, renormalize: bool = True, - num_fused_shared_experts: int = 0, custom_routing_function: Optional[Callable] = None, scoring_func: str = "softmax", correction_bias: Optional[torch.Tensor] = None, @@ -632,23 +632,21 @@ class TopK(BaseFusedOp): self.layer_id = layer_id - self.enable_waterfill = waterfill_routes_shared_experts( - num_fused_shared_experts - ) + shared_experts = get_model().shared_experts + self.enable_waterfill = shared_experts.use_waterfill self.waterfill_balancer = None - num_shared_slots = shared_topk_slots(num_fused_shared_experts) - num_routed_experts_per_token = top_k - num_fused_shared_experts + num_shared_slots = shared_experts.num_shared_topk_slots # Under the flashinfer_mxfp4 backend, fp4-expert ckpts take STANDARD # (consumes topk_ids/weights); otherwise BYPASSED. No-op on other backends. self.is_fp4_experts = is_fp4_experts self.topk_config = TopKConfig( - top_k=num_routed_experts_per_token + num_shared_slots, + top_k=top_k + num_shared_slots, use_grouped_topk=use_grouped_topk, renormalize=renormalize, topk_group=topk_group, num_expert_group=num_expert_group, - num_shared_topk_slots=num_shared_slots, + shared_experts=shared_experts, custom_routing_function=custom_routing_function, correction_bias=correction_bias, routed_scaling_factor=routed_scaling_factor, diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index ee9e798058..f7e346b189 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -9,6 +9,7 @@ from typing import NamedTuple import torch +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.environ import envs from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -500,12 +501,6 @@ def initialize_moe_config(): moe.tbo_token_distribution_threshold = overlap.tbo_token_distribution_threshold moe.disable_fp4_allgather = exec_moe.disable_flashinfer_cutlass_moe_fp4_allgather moe.quantization = get_model().quantization - # Seeded with the user's intent; each model's gate refines the ACTIVE - # value for its own build (install_shared_experts_fusion_decision). - moe.disable_shared_experts_fusion = exec_moe.disable_shared_experts_fusion - moe.speculative_disable_shared_experts_fusion = ( - exec_moe.disable_shared_experts_fusion - ) def get_moe_a2a_backend() -> MoeA2ABackend: @@ -543,29 +538,11 @@ def get_speculative_moe_a2a_backend() -> MoeA2ABackend: def is_shared_experts_fusion_disabled() -> bool: - """The ACTIVE shared-experts-fusion decision for the model being built. + return get_model().shared_experts.fusion_disabled - Written (both ways) by each MoE model's gate before its layers construct; - falls back to the config intent when no gate has run (models without an - auto-disable gate read the intent directly off the bag instead). - Construction-time only: a forward reads what its build baked in - (``num_fused_shared_experts`` on the layer). During a draft's build this - flag holds the DRAFT's decision, so a forward-time read would race the - build window — refuse it loudly.""" - from sglang.srt.model_executor.forward_context import has_forward_context - - if has_forward_context(): - raise AssertionError( - "is_shared_experts_fusion_disabled() called inside a forward: the " - "fusion decision is construction-time state (it can hold the draft's " - "value while a draft builds). Read the value your build baked in, " - "e.g. the layer's num_fused_shared_experts." - ) - moe = get_flags().moe - if moe.disable_shared_experts_fusion is None: - return get_exec().moe.disable_shared_experts_fusion - return moe.disable_shared_experts_fusion +def get_num_fused_shared_experts() -> int: + return get_model().shared_experts.num_fused_shared_experts @contextmanager @@ -574,9 +551,6 @@ def draft_model_build_scope(): settings; every value below returns to the target's on exit, including on error. - - Shared-experts fusion: the gates the draft runs also record their - decision on the speculative leaf; the target's ACTIVE - ``disable_shared_experts_fusion`` is restored. - ``boundary_reduction`` is ``--speculative-boundary-reduction``. Boundaries built here keep the resolved value after exit. - ``enable_w4a4_mxfp4_megamoe`` is ``--speculative-enable-w4a4-mxfp4-megamoe`` @@ -588,7 +562,6 @@ def draft_model_build_scope(): whole lifecycle (build + capture + forward), which not every worker does. """ moe = get_flags().moe - original_fusion = moe.disable_shared_experts_fusion original_scope = moe.in_speculative_scope try: moe.in_speculative_scope = True @@ -603,7 +576,6 @@ def draft_model_build_scope(): yield finally: moe.in_speculative_scope = original_scope - moe.disable_shared_experts_fusion = original_fusion def _draft_w4a4_mxfp4_megamoe_override() -> AbstractContextManager: @@ -615,36 +587,29 @@ def _draft_w4a4_mxfp4_megamoe_override() -> AbstractContextManager: return get_exec().moe.override(enable_w4a4_mxfp4_megamoe=draft_w4a4) -def install_shared_experts_fusion_decision( +def resolve_shared_experts_config( model_class, hf_config, quant_config -) -> None: - """Decide whether this runner's model fuses its shared experts, and install - the answer for the model it is about to build. - - Called from the loader's single model-instantiation point, so the decision - is made once per runner — before any layer exists — and the model classes - are pure readers (``is_shared_experts_fusion_disabled``). A model family - that can auto-disable exposes the conditions as - ``shared_experts_fusion_disable_reason(hf_config, quant_config)``; families - without one follow the user's intent. - - Inside ``draft_model_build_scope`` the answer also lands on the speculative - leaf, so a flags dump afterwards shows both runners' decisions. - """ - +) -> SharedExpertsConfig: + """Resolve a model layout before construction; fusion requires both hooks.""" disabled = get_exec().moe.disable_shared_experts_fusion + gate = getattr(model_class, "shared_experts_fusion_disable_reason", None) if not disabled: - gate = getattr(model_class, "shared_experts_fusion_disable_reason", None) reason = gate(hf_config, quant_config) if gate is not None else None if reason: log_info_on_rank0( logger, f"{reason} Shared experts fusion optimization is disabled." ) disabled = True - moe = get_flags().moe - moe.disable_shared_experts_fusion = disabled - if moe.in_speculative_scope: - moe.speculative_disable_shared_experts_fusion = disabled + count_fusable = getattr(model_class, "num_fusable_shared_experts", None) + if disabled or gate is None or count_fusable is None: + num_fused = 0 + else: + num_fused = count_fusable(hf_config) + return SharedExpertsConfig( + fusion_disabled=disabled, + num_fused_shared_experts=num_fused, + use_waterfill=num_fused > 0 and get_exec().moe.enable_waterfill, + ) def get_deepep_mode() -> DeepEPMode: @@ -699,24 +664,6 @@ def has_per_rank_fused_shared_slots(num_shared: int) -> bool: return num_shared > 0 and uses_per_rank_fused_shared_slots() -def waterfill_routes_shared_experts(num_fused_shared_experts: int) -> bool: - """Whether Waterfill, not the top-k router, writes the fused shared experts' column. - - Waterfill appends that column after the EPLB remap and may place it on another - rank, so the router it follows emits the routed columns only.""" - return num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill - - -def shared_topk_slots(num_fused_shared_experts: int) -> int: - """How many trailing top-k columns the router fills with fused shared experts. - - ``num_fused_shared_experts`` counts the shared experts the experts layer runs as - extra expert slots; the router writes their columns unless Waterfill does.""" - if waterfill_routes_shared_experts(num_fused_shared_experts): - return 0 - return num_fused_shared_experts - - def is_flashinfer_cutedsl_v1_path() -> bool: """CuteDSL v1 + DeepEP low-latency path (no MoeRunner, no autotune).""" return ( diff --git a/python/sglang/srt/model_executor/cpu_graph_runner.py b/python/sglang/srt/model_executor/cpu_graph_runner.py index f4a49b4fe0..ef61c22ee6 100644 --- a/python/sglang/srt/model_executor/cpu_graph_runner.py +++ b/python/sglang/srt/model_executor/cpu_graph_runner.py @@ -42,6 +42,7 @@ from sglang.srt.runtime_context import ( get_exec, get_flags, get_lora, + get_model, get_parallel, get_spec, ) @@ -734,6 +735,12 @@ class CPUGraphRunner: return is_bs_supported def capture(self) -> None: + with get_model().override( + shared_experts=self.model_runner.model.shared_experts_config + ): + self._capture() + + def _capture(self) -> None: capture_range = ( tqdm.tqdm(list(reversed(self.capture_bs))) if get_parallel().tp_rank == 0 diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 5d520d2428..506f3c45a4 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -1794,7 +1794,10 @@ class ModelRunner: forward_batch.split_index + forward_count, self.model_config.num_hidden_layers, ) - with device_timer_ctx(self.device_timer, "split_prefill"): + with ( + get_model().override(shared_experts=self.model.shared_experts_config), + device_timer_ctx(self.device_timer, "split_prefill"), + ): ret = self.model.forward_split_prefill( forward_batch.input_ids, forward_batch.positions, @@ -1968,7 +1971,10 @@ class ModelRunner: ctx_mgr = contextlib.nullcontext() else: ctx_mgr = forward_context(ForwardContext(attn_backend=self.attn_backend)) - with ctx_mgr: + with ( + get_model().override(shared_experts=self.model.shared_experts_config), + ctx_mgr, + ): mode_check = ( forward_batch.forward_mode.is_cpu_graph if self.device == "cpu" diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index 89674e457b..4b2c802eb2 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -51,6 +51,7 @@ from sglang.srt.runtime_context import ( get_disagg, get_exec, get_flags, + get_model, get_parallel, ) from sglang.srt.speculative.spec_info import create_dummy_verify_input @@ -243,7 +244,17 @@ class BaseRunner(ABC): self.attn_tp_size = get_parallel().attn_tp_size self.tbo_plugin = TboCudaGraphRunnerPlugin() + def model_context(self): + """Select the layout owned by this runner's current model.""" + return get_model().override( + shared_experts=self.model_runner.model.shared_experts_config + ) + def warmup(self) -> None: + with self.model_context(): + self._warmup() + + def _warmup(self) -> None: """Run kernel warmup + autotune once, gated by mr._kernel_warmed_up.""" mr = self.model_runner if getattr(mr, "_kernel_warmed_up", False): diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index fb837274ea..7793d76811 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -1107,6 +1107,10 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): return forward_batch, attn_backend, pp_proxy_tensors def capture(self) -> None: + with self.model_context(): + self._capture() + + def _capture(self) -> None: # Warm up + autotune kernels once before capture (run-once across the # decode + prefill runners; see BaseRunner.warmup). self.warmup() @@ -1593,7 +1597,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): shared_read_ends = self._resolve_shared_read_ends( self._replay_attn_backend(), forward_batch.forward_mode ) - with timer_ctx, self.backend.replay_session(): + with self.model_context(), timer_ctx, self.backend.replay_session(): self.load_batch(forward_batch, pp_proxy_tensors) if envs.SGLANG_LOG_DECODE_GRAPH_KEY.get(): logger.info( diff --git a/python/sglang/srt/model_executor/runner/eager_runner.py b/python/sglang/srt/model_executor/runner/eager_runner.py index f976b04034..d77170a3e1 100644 --- a/python/sglang/srt/model_executor/runner/eager_runner.py +++ b/python/sglang/srt/model_executor/runner/eager_runner.py @@ -209,20 +209,21 @@ class EagerRunner(BaseRunner): def execute( self, forward_batch: ForwardBatch, pp_proxy_tensors=None, **kwargs ) -> Any: - mode = forward_batch.forward_mode - if mode.is_mixed() and not is_npu() and get_cp_strategy() is None: - # A mixed batch is extend-shaped (decode tails are 1-token - # extends); run it as EXTEND. NPU keeps MIXED for its dedicated - # kernel; CP keeps it to skip the zigzag split. - forward_batch.forward_mode = ForwardMode.EXTEND - mode = ForwardMode.EXTEND - if mode.is_decode(): - return self._execute_decode(forward_batch, pp_proxy_tensors) - if mode.is_idle(): - return self._execute_idle(forward_batch, pp_proxy_tensors) - if mode.is_extend(include_draft_extend_v2=True): - return self._execute_extend(forward_batch, pp_proxy_tensors) - raise ValueError(f"Invalid forward mode for eager runner: {mode}") + with self.model_context(): + mode = forward_batch.forward_mode + if mode.is_mixed() and not is_npu() and get_cp_strategy() is None: + # A mixed batch is extend-shaped (decode tails are 1-token + # extends); run it as EXTEND. NPU keeps MIXED for its dedicated + # kernel; CP keeps it to skip the zigzag split. + forward_batch.forward_mode = ForwardMode.EXTEND + mode = ForwardMode.EXTEND + if mode.is_decode(): + return self._execute_decode(forward_batch, pp_proxy_tensors) + if mode.is_idle(): + return self._execute_idle(forward_batch, pp_proxy_tensors) + if mode.is_extend(include_draft_extend_v2=True): + return self._execute_extend(forward_batch, pp_proxy_tensors) + raise ValueError(f"Invalid forward mode for eager runner: {mode}") def _resolve_decode_pdmux( self, diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 3b2c75ab52..5cab65e43f 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -1657,6 +1657,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): return forward_batch, self.model_runner.attn_backend def capture(self) -> None: + with self.model_context(): + self._capture() + + def _capture(self) -> None: # Warm up + autotune kernels once before capture (run-once across the # decode + prefill runners; see BaseRunner.warmup). self.warmup() @@ -2307,7 +2311,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): self, forward_batch: ForwardBatch, **kwargs ) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]: self._validate_capture_hidden_mode(forward_batch) - with self.backend.replay_session(): + with self.model_context(), self.backend.replay_session(): static_forward_batch = self.load_batch(forward_batch, **kwargs) static_num_tokens = len(static_forward_batch.input_ids) raw_num_tokens = self.raw_num_tokens diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index 6d3a4183e3..fdb3187567 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -86,7 +86,7 @@ from sglang.srt.distributed import ( from sglang.srt.layers.layer_boundary.stage import check_stage_producers from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES from sglang.srt.layers.moe.utils import ( - install_shared_experts_fusion_decision, + resolve_shared_experts_config, ) from sglang.srt.layers.quantization.base_config import ( QuantizationConfig, @@ -285,11 +285,7 @@ def _initialize_model( ) -> nn.Module: """Initialize a model with the given configurations.""" model_class, _ = get_model_architecture(model_config) - # Decide the shared-experts-fusion question here, once per runner, before any - # layer exists: this is the only place a model class is instantiated, and it - # is the last point that still knows both the checkpoint's quantization and - # (through the build scope) whether this runner is a draft. - install_shared_experts_fusion_decision( + shared_experts = resolve_shared_experts_config( model_class, model_config.hf_config, quant_config ) kwargs = { @@ -305,8 +301,10 @@ def _initialize_model( if load_config.draft_model_idx is not None: kwargs["draft_model_idx"] = load_config.draft_model_idx - model = model_class(**kwargs) - check_stage_producers(model) + with get_model().override(shared_experts=shared_experts): + model = model_class(**kwargs) + check_stage_producers(model) + model.shared_experts_config = shared_experts return model diff --git a/python/sglang/srt/models/bailing_mm_v3.py b/python/sglang/srt/models/bailing_mm_v3.py index 982575bbdd..1b1db7fff4 100644 --- a/python/sglang/srt/models/bailing_mm_v3.py +++ b/python/sglang/srt/models/bailing_mm_v3.py @@ -68,6 +68,10 @@ class BailingMoeV3VLForConditionalGeneration(nn.Module): hf_config.text_config, quant_config ) + @staticmethod + def num_fusable_shared_experts(hf_config) -> int: + return BailingMoeV3ForCausalLM.num_fusable_shared_experts(hf_config.text_config) + def __init__( self, config: PretrainedConfig, diff --git a/python/sglang/srt/models/bailing_moe_nextn.py b/python/sglang/srt/models/bailing_moe_nextn.py index d85bf9d0cd..9770555ae6 100644 --- a/python/sglang/srt/models/bailing_moe_nextn.py +++ b/python/sglang/srt/models/bailing_moe_nextn.py @@ -32,6 +32,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor +from sglang.srt.layers.moe.utils import get_num_fused_shared_experts from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -71,7 +72,6 @@ class BailingMoEModelNextN(nn.Module): config: PretrainedConfig, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", - num_fused_shared_experts: int = 0, ) -> None: super().__init__() self.layer_group_size = 1 @@ -122,9 +122,6 @@ class BailingMoEModelNextN(nn.Module): } if _is_bailing_moe_v3_config(config): decoder_layer_cls = BailingMoeV3DecoderLayer - decoder_kwargs["num_fused_shared_experts"] = ( - num_fused_shared_experts - ) self.decoder = decoder_layer_cls(config, **decoder_kwargs) else: self.decoder = BailingMoEBlock( @@ -215,6 +212,12 @@ class BailingMoeForCausalLMNextN(nn.Module): expected_architecture="BailingMoeForCausalLMNextN", ) + @classmethod + def num_fusable_shared_experts(cls, hf_config) -> int: + if not _is_bailing_moe_v3_config(hf_config): + return 0 + return BailingMoeV3ForCausalLM.num_fusable_shared_experts(hf_config) + def __init__( self, config: PretrainedConfig, @@ -224,19 +227,16 @@ class BailingMoeForCausalLMNextN(nn.Module): nn.Module.__init__(self) self.config = config self.quant_config = quant_config - self.num_fused_shared_experts = 0 + # The weight loader reads this after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() is_bailing_moe_v3 = _is_bailing_moe_v3_config(config) if is_bailing_moe_v3: - BailingMoeV3ForCausalLM.determine_num_fused_shared_experts(self) - elif hasattr(self, "determine_num_fused_shared_experts"): - # Asystem has determine_num_fused_shared_experts but theta does not. - self.determine_num_fused_shared_experts("BailingMoeForCausalLMNextN") + BailingMoeV3ForCausalLM.log_shared_experts_fusion(self) self.model = BailingMoEModelNextN( config, quant_config, prefix=add_prefix("model", prefix), - num_fused_shared_experts=self.num_fused_shared_experts, ) self.lm_head = ParallelLMHead( config.vocab_size, diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 9f0989aaa9..411ce33dd0 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -48,7 +48,7 @@ from sglang.srt.layers.moe import ( from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk -from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled +from sglang.srt.layers.moe.utils import get_num_fused_shared_experts from sglang.srt.layers.multi_gate import ( create_multi_gate_mm_indices, multi_gate_triton_kernel, @@ -445,7 +445,6 @@ class BailingMoE(nn.Module): quant_config: Optional[QuantizationConfig] = None, layer_id: int = 0, prefix: str = "moe", - num_fused_shared_experts: int = 0, alt_stream: Optional[torch.cuda.Stream] = None, ): super().__init__() @@ -466,7 +465,7 @@ class BailingMoE(nn.Module): self.score_function = getattr(config, "score_function", None) self.multi_gate = is_bailing_multi_gate_enabled(config) - self.num_fused_shared_experts = num_fused_shared_experts + self.num_fused_shared_experts = get_num_fused_shared_experts() expert_swiglu_limit_list = getattr(config, "expert_swiglu_limit_list", None) share_expert_swiglu_limit_list = getattr( @@ -562,8 +561,7 @@ class BailingMoE(nn.Module): # Non-A2A EP (standard): each GPU computes shared expert, outputs are summed # via all_reduce → scale down by 1/ep_size to avoid double counting. # Note: A2A EP (DeepEP) + fused shared experts is impossible (expert routing - # breaks for the extra shared expert ID), so it's auto-disabled in - # determine_num_fused_shared_experts(). + # breaks for the extra shared expert ID), so the fusion veto disables it. fused_shared_experts_scaling_factor = None if self.moe_ep_size > 1 and self.num_fused_shared_experts > 0: fused_shared_experts_scaling_factor = 1.0 / float(self.moe_ep_size) @@ -581,7 +579,7 @@ class BailingMoE(nn.Module): gemm1_clamp_limit=self.expert_swiglu_limit, ) self.topk = TopK( - top_k=self.top_k + self.num_fused_shared_experts, + top_k=self.top_k, layer_id=self.layer_id, use_grouped_topk=self.use_grouped_topk, renormalize=self.norm_expert_prob, @@ -594,7 +592,6 @@ class BailingMoE(nn.Module): apply_routed_scaling_factor_on_output=( self.experts.should_fuse_routed_scaling_factor_in_topk ), - num_fused_shared_experts=self.num_fused_shared_experts, fused_shared_experts_scaling_factor=fused_shared_experts_scaling_factor, is_fp4_experts=( quant_config is not None @@ -1016,7 +1013,6 @@ class BailingMoELinearDecoderLayer(nn.Module): quant_config: Optional[QuantizationConfig] = None, layer_id: int = 0, prefix: str = "layer", - num_fused_shared_experts: int = 0, is_nextn: bool = False, alt_stream: Optional[torch.cuda.Stream] = None, ) -> None: @@ -1094,7 +1090,6 @@ class BailingMoELinearDecoderLayer(nn.Module): quant_config=quant_config, layer_id=self.layer_id, prefix=prefix, - num_fused_shared_experts=num_fused_shared_experts, alt_stream=alt_stream, ) else: @@ -1210,7 +1205,6 @@ class BailingMoELinearModel(nn.Module): config: PretrainedConfig, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", - num_fused_shared_experts: int = 0, ) -> None: super().__init__() self.pp_group = get_parallel().pp_group @@ -1258,7 +1252,6 @@ class BailingMoELinearModel(nn.Module): decoder_kwargs = { "quant_config": quant_config, "layer_id": layer_idx, - "num_fused_shared_experts": num_fused_shared_experts, } return BailingMoELinearDecoderLayer( layer_config, @@ -1394,14 +1387,14 @@ class BailingMoeV3ForCausalLM(nn.Module): self.config = config self.quant_config = quant_config - # Determine num_fused_shared_experts - self.determine_num_fused_shared_experts() + # The weight loader reads this after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() + self.log_shared_experts_fusion() self.model = BailingMoELinearModel( self.config, quant_config, prefix=add_prefix("model", prefix), - num_fused_shared_experts=self.num_fused_shared_experts, ) if self.pp_group.is_last_rank: @@ -1522,26 +1515,25 @@ class BailingMoeV3ForCausalLM(nn.Module): ) return None - def determine_num_fused_shared_experts(self): - self.num_fused_shared_experts = ( - 0 - if is_shared_experts_fusion_disabled() - else _get_bailing_num_shared_experts(self.config) - ) - if self.num_fused_shared_experts == 0: - return - + @classmethod + def num_fusable_shared_experts(cls, hf_config) -> int: + """Shared experts fused as routed slots when fusion is on.""" + num_shared_experts = _get_bailing_num_shared_experts(hf_config) # Safety check: current CUDA implementation only supports num_fused_shared_experts == 1. # The grouped_topk_gpu and _post_process_topk_ids functions only handle the last column, # which is incorrect when num_fused_shared_experts > 1. # AMD platform with aiter handles this correctly via fused_append_shared_experts kernel. - if self.num_fused_shared_experts > 1 and not _is_hip: + if num_shared_experts > 1 and not _is_hip: raise ValueError( - f"num_fused_shared_experts > 1 ({self.num_fused_shared_experts}) is not " + f"num_fused_shared_experts > 1 ({num_shared_experts}) is not " f"supported on CUDA platform. The current TopK implementation only handles " f"one fused shared expert. AMD platform with aiter supports multiple shared experts." ) + return num_shared_experts + def log_shared_experts_fusion(self): + if self.num_fused_shared_experts == 0: + return moe_ep_size = get_parallel().moe_ep_size if moe_ep_size > 1: log_info_on_rank0( diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index 462b2039c7..776a09b9d4 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -33,6 +33,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor +from sglang.srt.layers.moe.utils import get_num_fused_shared_experts from sglang.srt.layers.quantization import Fp8Config from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ( @@ -284,7 +285,7 @@ class DeepseekV3ForCausalLMNextN(DeepseekV3ForCausalLM): self.quant_config = quant_config # if not set, model load will be broken in DeepseekV3ForCausalLM load_weights() self.pp_group = get_parallel().pp_group - self.determine_num_fused_shared_experts() + self.num_fused_shared_experts = get_num_fused_shared_experts() nextn_quant_config = self._resolve_nextn_quant_config(config, quant_config) self.model = DeepseekModelNextN( diff --git a/python/sglang/srt/models/deepseek_ocr.py b/python/sglang/srt/models/deepseek_ocr.py index ba16f2441f..75a11e8adf 100644 --- a/python/sglang/srt/models/deepseek_ocr.py +++ b/python/sglang/srt/models/deepseek_ocr.py @@ -1421,21 +1421,33 @@ def build_qwen2_decoder_as_encoder( return decoder_as_encoder +def _fused_moe_text_config(hf_config): + """Return the nested fusable MoE config, or None for other branches.""" + text_config = hf_config.text_config + if is_ocr2_config(hf_config) or not ( + text_config.topk_method == "noaux_tc" or text_config.use_mla + ): + return None + return text_config + + class DeepseekOCRForCausalLM(nn.Module): @staticmethod def shared_experts_fusion_disable_reason(hf_config, quant_config): - text_config = hf_config.text_config - # Class-level hook: called before the model is built, so no `self.is_ocr2` yet. - if is_ocr2_config(hf_config) or not ( - text_config.topk_method == "noaux_tc" or text_config.use_mla - ): - # Those branches build the dense DeepseekForCausalLM, which has no - # shared experts to fuse. + text_config = _fused_moe_text_config(hf_config) + if text_config is None: return None return DeepseekV2ForCausalLM.shared_experts_fusion_disable_reason( text_config, quant_config ) + @staticmethod + def num_fusable_shared_experts(hf_config) -> int: + text_config = _fused_moe_text_config(hf_config) + if text_config is None: + return 0 + return DeepseekV2ForCausalLM.num_fusable_shared_experts(text_config) + def __init__( self, *, diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index bbde545b8d..47feb0773e 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -129,10 +129,10 @@ from sglang.srt.layers.moe.utils import ( RoutingMethodType, adds_replicated_output_to_partial, filter_moe_weight_param_global_expert, + get_num_fused_shared_experts, has_per_rank_fused_shared_slots, is_deepep_class_backend, is_sbo_enabled, - is_shared_experts_fusion_disabled, is_tbo_enabled, should_add_replicated_moe_output, sum_post_experts_output, @@ -676,11 +676,13 @@ class DeepseekV2MoE(nn.Module): n_shared_experts = ( 0 if config.n_shared_experts is None else int(config.n_shared_experts) ) - _fusion_disabled = is_shared_experts_fusion_disabled() - # num_fused_shared_experts drives weight remapping in deepseek_weight_loader: # mlp.shared_experts → mlp.experts.256 when > 0. - self.num_fused_shared_experts = 0 if _fusion_disabled else n_shared_experts + self.num_fused_shared_experts = get_num_fused_shared_experts() + assert self.num_fused_shared_experts in (0, n_shared_experts), ( + f"The model fuses {self.num_fused_shared_experts} shared experts, but " + f"this MoE block has {n_shared_experts}." + ) # DeepEP and MegaMOE shared expert fusion: shared expert is fused into # the same MoE kernel as a local expert at each EP rank. Expert layout @@ -762,9 +764,8 @@ class DeepseekV2MoE(nn.Module): if self.is_hash and not (is_nextn and is_deepseek_v4): self.topk = HashTopK( - topk=config.num_experts_per_tok + self.num_fused_shared_experts, + topk=config.num_experts_per_tok, num_experts=config.n_routed_experts, - num_fused_shared_experts=self.num_fused_shared_experts, vocab_size=config.vocab_size, scoring_func=config.scoring_func, routed_scaling_factor=self.routed_scaling_factor, @@ -778,12 +779,11 @@ class DeepseekV2MoE(nn.Module): else: # Default: grouped noaux_tc top-k. Covers V3/V3.2/GLM-5/Glm4MoeLite. topk_kwargs = dict( - top_k=config.num_experts_per_tok + self.num_fused_shared_experts, + top_k=config.num_experts_per_tok, layer_id=self.layer_id, renormalize=config.norm_topk_prob, use_grouped_topk=True, num_expert_group=config.n_group, - num_fused_shared_experts=self.num_fused_shared_experts, topk_group=config.topk_group, scoring_func=config.scoring_func, correction_bias=self.gate.e_score_correction_bias, @@ -3282,7 +3282,8 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): self.pp_group = get_parallel().pp_group self.config = config self.quant_config = quant_config - self.determine_num_fused_shared_experts() + # The weight loader reads this after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() self.use_dsa = is_deepseek_dsa(config) self.model = DeepseekV2Model( config, quant_config, prefix=add_prefix("model", prefix) @@ -3330,7 +3331,7 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): """Why this checkpoint cannot fuse its shared expert, or None. Evaluated by the loader once per runner, before any layer is built (see - ``install_shared_experts_fusion_decision``), so it takes the config and + ``resolve_shared_experts_config``), so it takes the config and quantization it is asked about rather than reading an instance. """ # Need to disable if quant precision mismatch, even if @@ -3397,11 +3398,10 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): return "Deepseek V3/R1 W4AFP8/W4A16 model uses different quant method for routed experts and shared experts." return None - def determine_num_fused_shared_experts(self): - # The decision was installed by the loader; this only reads it. - self.num_fused_shared_experts = ( - 0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts - ) + @classmethod + def num_fusable_shared_experts(cls, hf_config) -> int: + """Return the model's fusable shared-expert count.""" + return hf_config.n_shared_experts or 0 def get_input_embeddings(self) -> nn.Embedding: return self.model.embed_tokens diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 4099c05649..bc08449dde 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -90,7 +90,7 @@ from sglang.srt.layers.logits_processor import LogitsMetadata, LogitsProcessor from sglang.srt.layers.moe import get_moe_a2a_backend from sglang.srt.layers.moe.fused_moe_triton import FusedMoE from sglang.srt.layers.moe.utils import ( - is_shared_experts_fusion_disabled, + get_num_fused_shared_experts, should_skip_post_experts_all_reduce, uses_per_rank_fused_shared_slots, ) @@ -4447,7 +4447,8 @@ class DeepseekV4ForCausalLM(nn.Module): self.tp_size = get_parallel().tp_size self.quant_config = quant_config self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config) - self.determine_num_fused_shared_experts() + # The weight loader reads this after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() self.vision = None if ( config.model_type == "deepseek_v41" @@ -4663,11 +4664,10 @@ class DeepseekV4ForCausalLM(nn.Module): ) return None - def determine_num_fused_shared_experts(self): - # The decision was installed by the loader; this only reads it. - self.num_fused_shared_experts = ( - 0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts - ) + @classmethod + def num_fusable_shared_experts(cls, hf_config) -> int: + """Shared experts fused as routed slots when fusion is on.""" + return hf_config.n_shared_experts or 0 def prepare_language_model_inputs( self, diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index b1c0248806..33da73e761 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -24,7 +24,7 @@ from sglang.srt.layers.layer_boundary import layer_stack from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessorOutput -from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled +from sglang.srt.layers.moe.utils import get_num_fused_shared_experts from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.vocab_parallel_embedding import ( @@ -702,6 +702,10 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): hf_config, quant_config ) + @classmethod + def num_fusable_shared_experts(cls, hf_config) -> int: + return DeepseekV4ForCausalLM.num_fusable_shared_experts(hf_config) + def __init__( self, config: DeepSeekV4Config, @@ -711,9 +715,7 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): super().__init__() self.config = config self.quant_config = quant_config - self.num_fused_shared_experts = ( - 0 if is_shared_experts_fusion_disabled() else config.n_shared_experts - ) + self.num_fused_shared_experts = get_num_fused_shared_experts() dspark_config = parse_dspark_draft_config(draft_hf_config=config) if not dspark_config.require_markov(): diff --git a/python/sglang/srt/models/deepseek_v4_nextn.py b/python/sglang/srt/models/deepseek_v4_nextn.py index 3734c500ce..8fe278f965 100644 --- a/python/sglang/srt/models/deepseek_v4_nextn.py +++ b/python/sglang/srt/models/deepseek_v4_nextn.py @@ -24,7 +24,10 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor -from sglang.srt.layers.moe.utils import get_moe_a2a_backend +from sglang.srt.layers.moe.utils import ( + get_moe_a2a_backend, + get_num_fused_shared_experts, +) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.modelslim.modelslim import ModelSlimConfig from sglang.srt.layers.vocab_parallel_embedding import ( @@ -264,7 +267,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM): self.pp_group = get_parallel().pp_group self.quant_config = quant_config self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config) - self.determine_num_fused_shared_experts() + self.num_fused_shared_experts = get_num_fused_shared_experts() self.model = DeepseekV4ModelNextN( config, quant_config, prefix=add_prefix("model", prefix) diff --git a/python/sglang/srt/models/deepseek_vl2.py b/python/sglang/srt/models/deepseek_vl2.py index 8a56561ac9..c78bf8beac 100644 --- a/python/sglang/srt/models/deepseek_vl2.py +++ b/python/sglang/srt/models/deepseek_vl2.py @@ -156,17 +156,30 @@ class DeepseekVL2MlpProjector(nn.Module): return x +def _fused_moe_text_config(hf_config): + """Return the nested fusable MoE config, or None for other branches.""" + language_config = hf_config.language_config + return language_config if language_config.use_mla else None + + class DeepseekVL2ForCausalLM(nn.Module): @staticmethod def shared_experts_fusion_disable_reason(hf_config, quant_config): - language_config = hf_config.language_config - if not language_config.use_mla: + text_config = _fused_moe_text_config(hf_config) + if text_config is None: return None # The language model is built without a quantization config. return DeepseekV2ForCausalLM.shared_experts_fusion_disable_reason( - language_config, None + text_config, None ) + @staticmethod + def num_fusable_shared_experts(hf_config) -> int: + text_config = _fused_moe_text_config(hf_config) + if text_config is None: + return 0 + return DeepseekV2ForCausalLM.num_fusable_shared_experts(text_config) + def __init__( self, config: DeepseekVL2Config, diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 31b5e59f24..04821464c4 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -80,7 +80,7 @@ from sglang.srt.layers.moe import ( from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk -from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled +from sglang.srt.layers.moe.utils import get_num_fused_shared_experts from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.fp8_utils import ( block_quant_dequant, @@ -278,9 +278,7 @@ class Dots3MoE(nn.Module): super().__init__() self.tp_size = get_parallel().tp_size self.routed_scaling_factor = config.routed_scaling_factor - self.num_fused_shared_experts = ( - 0 if is_shared_experts_fusion_disabled() else config.n_shared_experts - ) + self.num_fused_shared_experts = get_num_fused_shared_experts() self.config = config self.layer_id = layer_id self.alt_stream = alt_stream @@ -314,12 +312,11 @@ class Dots3MoE(nn.Module): ) self.topk = TopK( - top_k=config.num_experts_per_tok + self.num_fused_shared_experts, + top_k=config.num_experts_per_tok, layer_id=self.layer_id, renormalize=config.norm_topk_prob, use_grouped_topk=True, num_expert_group=config.n_group, - num_fused_shared_experts=self.num_fused_shared_experts, topk_group=config.topk_group, correction_bias=self.gate.e_score_correction_bias, routed_scaling_factor=self.routed_scaling_factor, @@ -1605,7 +1602,8 @@ class Dots3LanguageModelForCausalLM(nn.Module): self.pp_group = get_parallel().pp_group self.config = config self.quant_config = quant_config - self.determine_num_fused_shared_experts() + # The weight loader reads this after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() self.model = Dots3Model( config, quant_config, prefix=add_prefix("model", prefix) ) @@ -1649,10 +1647,10 @@ class Dots3LanguageModelForCausalLM(nn.Module): ) return None - def determine_num_fused_shared_experts(self): - self.num_fused_shared_experts = ( - 0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts - ) + @classmethod + def num_fusable_shared_experts(cls, hf_config) -> int: + """Shared experts fused as routed slots when fusion is on.""" + return hf_config.n_shared_experts or 0 def get_input_embeddings(self) -> nn.Embedding: return self.model.embed_tokens @@ -2478,6 +2476,10 @@ class DotsNoteOmniForConditionalGeneration(nn.Module): hf_config, quant_config ) + @staticmethod + def num_fusable_shared_experts(hf_config) -> int: + return Dots3LanguageModelForCausalLM.num_fusable_shared_experts(hf_config) + def __init__( self, config: PretrainedConfig, diff --git a/python/sglang/srt/models/dots3_common/nextn.py b/python/sglang/srt/models/dots3_common/nextn.py index ea0281e751..8eaa929601 100644 --- a/python/sglang/srt/models/dots3_common/nextn.py +++ b/python/sglang/srt/models/dots3_common/nextn.py @@ -14,6 +14,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor +from sglang.srt.layers.moe.utils import get_num_fused_shared_experts from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -158,7 +159,7 @@ class Dots3NoteForCausalLMNextN(Dots3LanguageModelForCausalLM): "g_proj", ] } - self.determine_num_fused_shared_experts() + self.num_fused_shared_experts = get_num_fused_shared_experts() self.model = Dot3NoteModelNextN( config, quant_config, prefix=add_prefix("model", prefix) diff --git a/python/sglang/srt/models/dots_vlm.py b/python/sglang/srt/models/dots_vlm.py index 9a4ca86871..431827ac1f 100644 --- a/python/sglang/srt/models/dots_vlm.py +++ b/python/sglang/srt/models/dots_vlm.py @@ -38,17 +38,30 @@ from sglang.srt.runtime_context import get_parallel from .dots_vlm_vit import DotsVisionTransformer +def _fused_moe_text_config(hf_config): + """Return the nested fusable MoE config, or None for other branches.""" + return None if hf_config.encoder_only else hf_config.language_config + + class DotsVLMForCausalLM(nn.Module): """DotsVLM model for sglang inference""" @staticmethod def shared_experts_fusion_disable_reason(hf_config, quant_config): - if hf_config.encoder_only: + text_config = _fused_moe_text_config(hf_config) + if text_config is None: return None return DeepseekV2ForCausalLM.shared_experts_fusion_disable_reason( - hf_config.language_config, quant_config + text_config, quant_config ) + @staticmethod + def num_fusable_shared_experts(hf_config) -> int: + text_config = _fused_moe_text_config(hf_config) + if text_config is None: + return 0 + return DeepseekV2ForCausalLM.num_fusable_shared_experts(text_config) + def __init__( self, config: DotsVLMConfig, quant_config: Optional[QuantizationConfig] = None ) -> None: diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 2b6ff441a0..e54a6894ab 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -69,7 +69,7 @@ from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_t from sglang.srt.layers.moe.utils import ( RoutingMethodType, filter_moe_weight_param_global_expert, - is_shared_experts_fusion_disabled, + get_num_fused_shared_experts, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention @@ -414,9 +414,7 @@ class Glm4MoeSparseMoeBlock(nn.Module): self.tp_size = get_parallel().tp_size self.routed_scaling_factor = config.routed_scaling_factor self.n_shared_experts = config.n_shared_experts - self.num_fused_shared_experts = ( - 0 if is_shared_experts_fusion_disabled() else config.n_shared_experts - ) + self.num_fused_shared_experts = get_num_fused_shared_experts() self.config = config self.layer_id = layer_id @@ -450,7 +448,7 @@ class Glm4MoeSparseMoeBlock(nn.Module): ) self.topk = TopK( - top_k=self.top_k + self.num_fused_shared_experts, + top_k=self.top_k, layer_id=self.layer_id, renormalize=config.norm_topk_prob, use_grouped_topk=True, @@ -458,7 +456,6 @@ class Glm4MoeSparseMoeBlock(nn.Module): topk_group=config.topk_group, correction_bias=self.gate.e_score_correction_bias, routed_scaling_factor=self.routed_scaling_factor, - num_fused_shared_experts=self.num_fused_shared_experts, apply_routed_scaling_factor_on_output=getattr( self.experts, "should_fuse_routed_scaling_factor_in_topk", False ), @@ -995,8 +992,8 @@ class Glm4MoeForCausalLM(nn.Module): self.pp_group = get_parallel().pp_group self.config = config self.quant_config = quant_config - self.num_fused_shared_experts = 0 - self.determine_num_fused_shared_experts() + # The weight loader reads this after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() self.model = Glm4MoeModel( config, quant_config, prefix=add_prefix("model", prefix) ) @@ -1033,11 +1030,10 @@ class Glm4MoeForCausalLM(nn.Module): return "GLM-4.5 W4AFP8 model uses different quant method for routed experts and shared experts." return None - def determine_num_fused_shared_experts(self): - # The decision was installed by the loader; this only reads it. - self.num_fused_shared_experts = ( - 0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts - ) + @classmethod + def num_fusable_shared_experts(cls, hf_config) -> int: + """Shared experts fused as routed slots when fusion is on.""" + return hf_config.n_shared_experts or 0 def get_input_embeddings(self) -> nn.Embedding: return self.model.embed_tokens diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 7fcf314333..c8f026f399 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -66,7 +66,7 @@ from sglang.srt.layers.moe.topk import ( ) from sglang.srt.layers.moe.utils import ( filter_moe_weight_param_global_expert, - is_shared_experts_fusion_disabled, + get_num_fused_shared_experts, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.utils import PPMissingLayer @@ -185,9 +185,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): self.tp_size = get_parallel().tp_size self.routed_scaling_factor = config.routed_scaling_factor self.n_shared_experts = config.n_shared_experts - self.num_fused_shared_experts = ( - 0 if is_shared_experts_fusion_disabled() else config.n_shared_experts - ) + self.num_fused_shared_experts = get_num_fused_shared_experts() self.config = config self.layer_id = layer_id self.alt_stream = alt_stream @@ -224,12 +222,11 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): ) self.topk = TopK( - top_k=config.num_experts_per_tok + self.num_fused_shared_experts, + top_k=config.num_experts_per_tok, layer_id=self.layer_id, renormalize=config.norm_topk_prob, use_grouped_topk=True, num_expert_group=config.n_group, - num_fused_shared_experts=self.num_fused_shared_experts, topk_group=config.topk_group, correction_bias=self.gate.e_score_correction_bias, quant_config=quant_config, @@ -704,7 +701,8 @@ class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): self.config = config self.quant_config = quant_config self.pp_group = get_parallel().pp_group - self.determine_num_fused_shared_experts() + # The weight loader reads this after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() self.model = Glm4MoeLiteModel( config, quant_config, prefix=add_prefix("model", prefix) ) @@ -749,11 +747,10 @@ class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): return "GLM-4.5 or GLM-4.6 cannot use shared experts fusion optimization under expert parallelism." return None - def determine_num_fused_shared_experts(self): - # The decision was installed by the loader; this only reads it. - self.num_fused_shared_experts = ( - 0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts - ) + @classmethod + def num_fusable_shared_experts(cls, hf_config) -> int: + """Shared experts fused as routed slots when fusion is on.""" + return hf_config.n_shared_experts or 0 def get_input_embeddings(self) -> nn.Embedding: return self.model.embed_tokens diff --git a/python/sglang/srt/models/glm4_moe_lite_nextn.py b/python/sglang/srt/models/glm4_moe_lite_nextn.py index e63c6e7fc8..e6607a6aa0 100644 --- a/python/sglang/srt/models/glm4_moe_lite_nextn.py +++ b/python/sglang/srt/models/glm4_moe_lite_nextn.py @@ -27,6 +27,7 @@ from sglang.srt.layers.layer_boundary import layer_stack from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor +from sglang.srt.layers.moe.utils import get_num_fused_shared_experts from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -148,10 +149,8 @@ class Glm4MoeLiteForCausalLMNextN(Glm4MoeLiteForCausalLM): quant_config = None self.quant_config = quant_config - # The draft's own gate (its quantization can differ from the - # target's); the decoder below reads the ACTIVE decision as it builds, - # and num_fused_shared_experts drives the inherited loader's remap. - self.determine_num_fused_shared_experts() + # Drives the inherited loader's shared-expert remap after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() self.model = Glm4MoeLiteModelNextN( config, quant_config, prefix=add_prefix("model", prefix) diff --git a/python/sglang/srt/models/glm4_moe_nextn.py b/python/sglang/srt/models/glm4_moe_nextn.py index d35d2a772c..a75911993c 100644 --- a/python/sglang/srt/models/glm4_moe_nextn.py +++ b/python/sglang/srt/models/glm4_moe_nextn.py @@ -27,6 +27,7 @@ from sglang.srt.layers.layer_boundary import layer_stack from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor +from sglang.srt.layers.moe.utils import get_num_fused_shared_experts from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -129,12 +130,8 @@ class Glm4MoeForCausalLMNextN(Glm4MoeForCausalLM): quant_config = None self.quant_config = quant_config - # The draft's own gate: its quantization can differ from the - # target's, and the decoder below reads the ACTIVE decision while it - # builds. Also sets num_fused_shared_experts, which drives the - # inherited loader's shared-expert remap. - self.num_fused_shared_experts = 0 - self.determine_num_fused_shared_experts() + # Drives the inherited loader's shared-expert remap after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() self.model = Glm4MoeModelNextN( config, quant_config, prefix=add_prefix("model", prefix) diff --git a/python/sglang/srt/models/glm4v_moe.py b/python/sglang/srt/models/glm4v_moe.py index 842cf52e4e..ed97a7f412 100644 --- a/python/sglang/srt/models/glm4v_moe.py +++ b/python/sglang/srt/models/glm4v_moe.py @@ -10,7 +10,7 @@ from sglang.srt.layers.attention import vision_utils from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import get_moe_a2a_backend from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE -from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled +from sglang.srt.layers.moe.utils import get_num_fused_shared_experts from sglang.srt.layers.pooler import Pooler, PoolingType from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.utils import PPMissingLayer @@ -44,8 +44,10 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): self.use_data_parallel = get_mm().mm_enable_dp_encoder vision_utils.update_vit_attn_dummy_heads_config(self.config) self.quant_config = quant_config - self.num_fused_shared_experts = 0 - self.determine_num_fused_shared_experts() + # The weight loader reads this after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() + if self.num_fused_shared_experts: + log_info_on_rank0(logger, "Shared experts fusion optimization enabled.") self.model = Glm4MoeModel( config, @@ -97,15 +99,13 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): return "Shared experts fusion is not supported when Deepep MoE backend is enabled." return None - def determine_num_fused_shared_experts(self): - # The decision was installed by the loader; this only reads it. - if is_shared_experts_fusion_disabled(): - return - self.num_fused_shared_experts = self.config.n_shared_experts - assert self.num_fused_shared_experts == 1, ( + @classmethod + def num_fusable_shared_experts(cls, hf_config) -> int: + """Shared experts fused as routed slots when fusion is on.""" + assert hf_config.n_shared_experts == 1, ( "Only 1 fused shared expert is supported for Glm4vMoeForConditionalGeneration" ) - log_info_on_rank0(logger, "Shared experts fusion optimization enabled.") + return hf_config.n_shared_experts def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]], is_nextn=False): if is_nextn: diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index ecc8197f1e..32f0db2552 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -54,7 +54,7 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.utils import ( get_moe_a2a_backend, get_moe_runner_backend, - is_shared_experts_fusion_disabled, + get_num_fused_shared_experts, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.utils import is_layer_skipped @@ -1277,7 +1277,10 @@ class Glm5NextForConditionalGeneration(nn.Module): self.lm_head = None self.logits_processor = None if not self.encoder_only: - self.determine_num_fused_shared_experts() + # The weight loader reads this after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() + if self.num_fused_shared_experts: + log_info_on_rank0(logger, "Shared experts fusion optimization enabled.") self.model = Glm5NextModel( text_config, quant_config, prefix=add_prefix("model", prefix) ) @@ -1491,16 +1494,14 @@ class Glm5NextForConditionalGeneration(nn.Module): return reason return None - def determine_num_fused_shared_experts(self): - self.num_fused_shared_experts = ( - 0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts + @classmethod + def num_fusable_shared_experts(cls, hf_config) -> int: + """Return the model's fusable shared-expert count.""" + num_shared_experts = hf_config.text_config.n_shared_experts + assert num_shared_experts == 1, ( + f"Only 1 fused shared expert is supported for {cls.__name__}" ) - if self.num_fused_shared_experts == 0: - return - assert self.num_fused_shared_experts == 1, ( - f"Only 1 fused shared expert is supported for {type(self).__name__}" - ) - log_info_on_rank0(logger, "Shared experts fusion optimization enabled.") + return num_shared_experts def wants_prefill_autotune(self) -> bool: backend = get_moe_runner_backend() diff --git a/python/sglang/srt/models/kimi_k25.py b/python/sglang/srt/models/kimi_k25.py index fb566b090b..41c0a262d5 100644 --- a/python/sglang/srt/models/kimi_k25.py +++ b/python/sglang/srt/models/kimi_k25.py @@ -632,6 +632,11 @@ def mm_projection_auto( return projected.reshape(-1, projected.shape[-1]) +def _fused_moe_text_config(hf_config): + """Return the nested fusable MoE config, or None for other branches.""" + return None if hf_config.encoder_only else hf_config.text_config + + class KimiK25ForConditionalGeneration(nn.Module): supports_cuda_vmm_feature_transport = True @@ -646,12 +651,20 @@ class KimiK25ForConditionalGeneration(nn.Module): @staticmethod def shared_experts_fusion_disable_reason(hf_config, quant_config): - if hf_config.encoder_only: + text_config = _fused_moe_text_config(hf_config) + if text_config is None: return None return DeepseekV3ForCausalLM.shared_experts_fusion_disable_reason( - hf_config.text_config, quant_config + text_config, quant_config ) + @staticmethod + def num_fusable_shared_experts(hf_config) -> int: + text_config = _fused_moe_text_config(hf_config) + if text_config is None: + return 0 + return DeepseekV3ForCausalLM.num_fusable_shared_experts(text_config) + def __init__( self, config: KimiK25Config, diff --git a/python/sglang/srt/models/kimi_vl.py b/python/sglang/srt/models/kimi_vl.py index 267b36994a..8703e8ccee 100644 --- a/python/sglang/srt/models/kimi_vl.py +++ b/python/sglang/srt/models/kimi_vl.py @@ -120,15 +120,28 @@ def _language_model_config(config: KimiVLConfig): return text_config +def _fused_moe_text_config(hf_config): + """Return the nested fusable MoE config, or None for other branches.""" + return None if hf_config.encoder_only else _language_model_config(hf_config) + + class KimiVLForConditionalGeneration(nn.Module): @staticmethod def shared_experts_fusion_disable_reason(hf_config, quant_config): - if hf_config.encoder_only: + text_config = _fused_moe_text_config(hf_config) + if text_config is None: return None return DeepseekV2ForCausalLM.shared_experts_fusion_disable_reason( - _language_model_config(hf_config), quant_config + text_config, quant_config ) + @staticmethod + def num_fusable_shared_experts(hf_config) -> int: + text_config = _fused_moe_text_config(hf_config) + if text_config is None: + return 0 + return DeepseekV2ForCausalLM.num_fusable_shared_experts(text_config) + def __init__( self, config: KimiVLConfig, diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 2653380e9a..c388a31135 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -67,7 +67,7 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK from sglang.srt.layers.moe.utils import ( get_moe_a2a_backend, - is_shared_experts_fusion_disabled, + get_num_fused_shared_experts, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.mxfp8_input import ( @@ -384,9 +384,7 @@ class MiniMaxM3MoE(nn.Module): self.tp_size = get_parallel().tp_size self.alt_stream = alt_stream self.n_shared_experts = getattr(config, "n_shared_experts", None) - self.num_fused_shared_experts = ( - 0 if is_shared_experts_fusion_disabled() else config.n_shared_experts - ) + self.num_fused_shared_experts = get_num_fused_shared_experts() if self.tp_size > config.num_local_experts: raise ValueError( @@ -424,12 +422,11 @@ class MiniMaxM3MoE(nn.Module): gate_up_interleaved=False, ) self.topk = TopK( - top_k=config.num_experts_per_tok + self.num_fused_shared_experts, + top_k=config.num_experts_per_tok, renormalize=True, layer_id=layer_id, scoring_func=config.scoring_func, correction_bias=self.e_score_correction_bias, - num_fused_shared_experts=self.num_fused_shared_experts, routed_scaling_factor=self.routed_scaling_factor, apply_routed_scaling_factor_on_output=True, ) @@ -1763,8 +1760,10 @@ class MiniMaxM3SparseForCausalLM(nn.Module): self.quant_config = quant_config self.pp_group = get_parallel().pp_group - self.num_fused_shared_experts = 0 - self.determine_num_fused_shared_experts() + # The weight loader reads this after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() + if self.num_fused_shared_experts: + log_info_on_rank0(logger, "Shared experts fusion optimization enabled.") self.model = MiniMaxM3Model( config, quant_config, prefix=add_prefix("model", prefix) @@ -1811,15 +1810,13 @@ class MiniMaxM3SparseForCausalLM(nn.Module): return "Shared experts fusion is not supported when Deepep MoE backend is enabled." return None - def determine_num_fused_shared_experts(self): - # The decision was installed by the loader; this only reads it. - if is_shared_experts_fusion_disabled(): - return - self.num_fused_shared_experts = self.config.n_shared_experts - assert self.num_fused_shared_experts == 1, ( + @classmethod + def num_fusable_shared_experts(cls, hf_config) -> int: + """Shared experts fused as routed slots when fusion is on.""" + assert hf_config.n_shared_experts == 1, ( "Only 1 fused shared expert is supported for MiniMax-M3" ) - log_info_on_rank0(logger, "Shared experts fusion optimization enabled.") + return hf_config.n_shared_experts def set_eagle3_layers_to_capture(self, layer_ids: Optional[list[int]] = None): if not self.pp_group.is_last_rank: diff --git a/python/sglang/srt/models/minimax_m3_vl.py b/python/sglang/srt/models/minimax_m3_vl.py index 82fcd5bbf4..f6f5ee1a84 100644 --- a/python/sglang/srt/models/minimax_m3_vl.py +++ b/python/sglang/srt/models/minimax_m3_vl.py @@ -10,7 +10,7 @@ import torch.nn as nn from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.utils import ( get_moe_a2a_backend, - is_shared_experts_fusion_disabled, + get_num_fused_shared_experts, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.utils import PPMissingLayer @@ -88,8 +88,10 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module): self.use_data_parallel = get_mm().mm_enable_dp_encoder - self.num_fused_shared_experts = 0 - self._determine_num_fused_shared_experts() + # The weight loader reads this after the build. + self.num_fused_shared_experts = get_num_fused_shared_experts() + if self.num_fused_shared_experts: + log_info_on_rank0(logger, "Shared experts fusion optimization enabled.") vision_config_raw = config.vision_config assert vision_config_raw is not None, "vision_config is required" @@ -177,15 +179,12 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module): ) return None - def _determine_num_fused_shared_experts(self) -> None: - # The decision was installed by the loader; this only reads it. - if is_shared_experts_fusion_disabled(): - return - self.num_fused_shared_experts = self.config.text_config.n_shared_experts - assert self.num_fused_shared_experts == 1, ( - "Only 1 fused shared expert is supported" - ) - log_info_on_rank0(logger, "Shared experts fusion optimization enabled.") + @classmethod + def num_fusable_shared_experts(cls, hf_config) -> int: + """Return the model's fusable shared-expert count.""" + num_shared_experts = hf_config.text_config.n_shared_experts + assert num_shared_experts == 1, "Only 1 fused shared expert is supported" + return num_shared_experts @classmethod def get_model_config_for_expert_location(cls, config): diff --git a/python/sglang/srt/models/pixtral.py b/python/sglang/srt/models/pixtral.py index 95dd5a7e43..579c36d902 100644 --- a/python/sglang/srt/models/pixtral.py +++ b/python/sglang/srt/models/pixtral.py @@ -68,19 +68,33 @@ class VisionEncoderArgs: mm_projector_id: str = "" +def _fused_moe_text_config(hf_config): + """Return the nested fusable MoE config, or None for other branches.""" + text_config = hf_config.text_config + if getattr(text_config, "model_type", "") != "deepseek_v3": + return None + return text_config + + class PixtralForConditionalGeneration(nn.Module): merge_by_field_config = True @staticmethod def shared_experts_fusion_disable_reason(hf_config, quant_config): - text_config = hf_config.text_config - if getattr(text_config, "model_type", "") != "deepseek_v3": - # The GQA text config builds the dense Mistral backbone. + text_config = _fused_moe_text_config(hf_config) + if text_config is None: return None return MistralLarge3ForCausalLM.shared_experts_fusion_disable_reason( text_config, quant_config ) + @staticmethod + def num_fusable_shared_experts(hf_config) -> int: + text_config = _fused_moe_text_config(hf_config) + if text_config is None: + return 0 + return MistralLarge3ForCausalLM.num_fusable_shared_experts(text_config) + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("image"): diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 0e64489406..e17d01a4fe 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -557,19 +557,7 @@ class MoeFlags(_FlagGroupBase): tbo_token_distribution_threshold: float | None = None disable_fp4_allgather: bool | None = None quantization: str | None = None - # The shared-experts-fusion decision, per runner — the runner_backend / - # speculative_runner_backend shape. Both leaves are seeded from the config - # intent by ``initialize_moe_config``; each MoE model's gate - # (determine_num_fused_shared_experts) refines the ACTIVE leaf, both ways, - # before its layers build and read it. ``speculative_moe_backend_context`` - # brackets a draft's build: on exit the draft's effective decision is - # persisted onto the speculative leaf (inspectable afterwards) and the - # target's ACTIVE value returns. - disable_shared_experts_fusion: bool | None = None - speculative_disable_shared_experts_fusion: bool | None = None - # Lifecycle marker (the capture.disable_dispose_tensor family): set while - # speculative_moe_backend_context is active, so a draft gate's write also - # lands on the speculative leaf. + # True while draft construction/backend scopes are active. in_speculative_scope: bool = False # Draft construction/execution uses a separate one-sided A2A workspace from # the target model's concurrently live CUDA graphs. diff --git a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py index 4a5ec748d4..85caa24e9f 100644 --- a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py @@ -300,7 +300,8 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner): self._metadata_glue = None def _replay_graph(self, shape_key, forward_batch): - return self.backend.replay(shape_key, forward_batch) + with self.model_context(): + return self.backend.replay(shape_key, forward_batch) # ----------------------------------------------------------------- # Helpers diff --git a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py index ddc5bd0d95..663fd2773e 100644 --- a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py @@ -307,7 +307,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): ) def _replay_graph(self, shape_key, forward_batch): - return self.backend.replay(shape_key, forward_batch) + with self.model_context(): + return self.backend.replay(shape_key, forward_batch) def _cache_loc_dtype(self): return torch.int64 diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py b/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py index 15733aba40..773abb1650 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py @@ -203,7 +203,8 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner): return ShapeKey(size=bs) def _replay_graph(self, shape_key, forward_batch): - return self.backend.replay(shape_key, forward_batch) + with self.model_context(): + return self.backend.replay(shape_key, forward_batch) def can_run_graph(self, forward_batch: ForwardBatch): # Uniform-width replay invariant: the batch's actual per-request width diff --git a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py index 6c67b1b471..3a92275839 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py @@ -227,7 +227,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): ) def _replay_graph(self, shape_key, forward_batch): - return self.backend.replay(shape_key, forward_batch) + with self.model_context(): + return self.backend.replay(shape_key, forward_batch) def _make_graph_key(self, bs, stream_idx=None, variant_label=None): return ShapeKey(size=bs) @@ -412,11 +413,12 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): output_cache_loc_backup = forward_batch.out_cache_loc hidden_states_backup = forward_batch.spec_info.hidden_states - ret = self.model_runner.model.forward( - forward_batch.input_ids, - forward_batch.positions, - forward_batch, - ) + with self.model_context(): + ret = self.model_runner.model.forward( + forward_batch.input_ids, + forward_batch.positions, + forward_batch, + ) if self.eagle_worker.chain_mtp_hidden_states and ret.hidden_states is not None: buffers.hidden_states[:num_tokens].copy_(ret.hidden_states[:num_tokens]) @@ -1083,7 +1085,8 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner( with device_timer_ctx( first.model_runner.device_timer, "eagle_draft_extend" ): - outs = first.backend.replay(shape_key, self._replay_spec_info) + with first.model_context(): + outs = first.backend.replay(shape_key, self._replay_spec_info) raw_bs = self.raw_bs self._cached = {} non_null = [r for r in self.runners if r is not None] diff --git a/python/sglang/srt/state_capturer/routed_experts.py b/python/sglang/srt/state_capturer/routed_experts.py index b7913a9f9e..f8516736a0 100644 --- a/python/sglang/srt/state_capturer/routed_experts.py +++ b/python/sglang/srt/state_capturer/routed_experts.py @@ -49,9 +49,9 @@ class RoutedExpertsCapturer(BaseTopkCapturer): ) -> Optional["RoutedExpertsCapturer"]: if not get_exec().features.enable_return_routed_experts: return None - # The model's own attribute is the baked decision (0 when its gate - # disabled fusion); the ACTIVE flag can be holding another runner's. - num_fused_shared_experts = getattr(model, "num_fused_shared_experts", 0) + # The loader attaches this to every entry model, including wrappers. + # Size from the owner, independent of which runner is currently active. + num_fused_shared_experts = model.shared_experts_config.num_fused_shared_experts return RoutedExpertsCapturer( model_config, num_tokens=num_tokens, diff --git a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py index 2f562fd907..1c41428fc5 100644 --- a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py +++ b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py @@ -8,6 +8,7 @@ from unittest.mock import patch import torch from torch import nn +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, @@ -672,6 +673,7 @@ class _EagleDraftExtendForward(nn.Module): device: str, ): super().__init__() + self.shared_experts_config = SharedExpertsConfig() self.module = module self.token_embed = nn.Embedding( vocab_size, hidden_size, dtype=dtype, device=device diff --git a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py index 23fcb5ef1f..26041625c8 100644 --- a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py +++ b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py @@ -7,6 +7,7 @@ from unittest.mock import patch import torch from torch import nn +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseConfig from sglang.srt.model_executor.forward_batch_info import ( @@ -153,6 +154,10 @@ class _DummyTpGroup: class _TinyDraftModel(nn.Module): + def __init__(self): + super().__init__() + self.shared_experts_config = SharedExpertsConfig() + def forward(self, *args, **kwargs): raise RuntimeError("EAGLEDraftCudaGraphRunner should call worker.draft_forward") diff --git a/test/registered/kernels/ops/moe/test_jit_grouped_topk.py b/test/registered/kernels/ops/moe/test_jit_grouped_topk.py index d48a7015ea..9b24925e96 100644 --- a/test/registered/kernels/ops/moe/test_jit_grouped_topk.py +++ b/test/registered/kernels/ops/moe/test_jit_grouped_topk.py @@ -16,6 +16,7 @@ import sys import pytest import torch +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") @@ -37,7 +38,7 @@ def _jit_routed(monkeypatch, logits, hidden, bias, groups): renormalize=True, use_grouped_topk=True, num_expert_group=groups, - num_shared_topk_slots=SHARED, + shared_experts=SharedExpertsConfig(num_fused_shared_experts=SHARED), topk_group=1, scoring_func="sigmoid", correction_bias=bias, diff --git a/test/registered/kernels/ops/moe/test_rocm_router_gate.py b/test/registered/kernels/ops/moe/test_rocm_router_gate.py index 37e2f7483e..4526877ff3 100644 --- a/test/registered/kernels/ops/moe/test_rocm_router_gate.py +++ b/test/registered/kernels/ops/moe/test_rocm_router_gate.py @@ -10,6 +10,7 @@ from sglang.kernels.ops.gemm.router_gemv_hip import ( rocm_router_reduce_partials, ) from sglang.kernels.ops.moe.rocm_router_gate import rocm_router_gate +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci from sglang.test.test_utils import CustomTestCase @@ -135,7 +136,7 @@ class TestRocmRouterGate(CustomTestCase): config = TopKConfig( top_k=TOPK + 1, renormalize=True, - num_shared_topk_slots=1, + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1), correction_bias=self.bias_bf16, routed_scaling_factor=1.5, fused_shared_experts_scaling_factor=scaling, diff --git a/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py b/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py index 9c224929ba..29c3800b0b 100644 --- a/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py +++ b/test/registered/kernels/ops/moe/test_round_robin_ep_balance.py @@ -6,6 +6,7 @@ from unittest.mock import patch import pytest import torch +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.environ import envs from sglang.srt.layers.moe.topk import ( TopKConfig, @@ -227,7 +228,9 @@ def _select( router_logits=torch.zeros((num_tokens, NUM_EXPERTS), device="cuda"), topk_config=TopKConfig( top_k=top_k, - num_shared_topk_slots=num_fused_shared_experts, + shared_experts=SharedExpertsConfig( + num_fused_shared_experts=num_fused_shared_experts + ), custom_routing_function=_fixed_router, ), layer_id=0, diff --git a/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py b/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py index b817243680..a0394415bc 100644 --- a/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py +++ b/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py @@ -7,22 +7,19 @@ GPU-only (Triton), so these tests are skipped when no accelerator is present. """ import unittest -from unittest.mock import patch import torch from sglang.kernels.ops.moe.fused_moe_triton_kernels import ( fused_append_remap_shared_experts_deepep, ) -from sglang.srt.layers.moe import topk as topk_module +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.layers.moe.topk import ( TopKConfig, - biased_grouped_topk_impl, remap_topk_for_per_rank_shared_slots, - select_experts, ) -from sglang.srt.runtime_context import get_context, get_parallel -from sglang.srt.utils import get_device, is_cuda +from sglang.srt.runtime_context import get_parallel +from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -210,7 +207,9 @@ class TestFusedAppendRemapPerRankSharedSlots(CustomTestCase): npr, TopKConfig( top_k=k, - num_shared_topk_slots=s, + shared_experts=SharedExpertsConfig( + num_fused_shared_experts=s + ), routed_scaling_factor=rsf, apply_routed_scaling_factor_on_output=folded, ), @@ -289,160 +288,5 @@ class TestFusedAppendRemapPerRankSharedSlots(CustomTestCase): self.assertTrue(torch.equal(got_w, topk_weights)) -@unittest.skipUnless(is_cuda(), "covers the CUDA routers") -class TestSelectExpertsPerRankSharedSlots(CustomTestCase): - """Shared slots must not alter routed selection or normalization.""" - - NUM_TOKENS, TOP_K, EP_SIZE, EP_RANK = 32, 8, 8, 3 - # (num_experts, num_expert_group, topk_group, scoring_func, biased); each row - # reaches a different router. - ROUTERS = [ - (256, 8, 4, "sigmoid", True), # DeepSeek-V3 grouped routing - (384, 8, 4, "sigmoid", True), # grouped, outside FlashInfer's shapes - (64, 8, 3, "softmax", False), # grouped, no correction bias - (256, None, None, "sqrtsoftplus", True), # DeepSeek-V4 routing - (128, None, None, "softmax", False), - ] - - def _route( - self, *, num_fused_shared_experts, logits, bias, router, renormalize=True - ): - _, num_expert_group, topk_group, scoring_func, _ = router - topk_config = TopKConfig( - top_k=self.TOP_K + num_fused_shared_experts, - renormalize=renormalize, - use_grouped_topk=num_expert_group is not None, - num_expert_group=num_expert_group, - topk_group=topk_group, - num_shared_topk_slots=num_fused_shared_experts, - correction_bias=bias, - scoring_func=scoring_func, - routed_scaling_factor=2.5, - allow_routed_experts_capture=False, - ) - with ( - patch.object( - topk_module, - "has_per_rank_fused_shared_slots", - side_effect=lambda num_fused: num_fused > 0, - ), - get_parallel().override(moe_ep_size=self.EP_SIZE, moe_ep_rank=self.EP_RANK), - ): - out = select_experts( - torch.empty(logits.shape[0], 16, device=logits.device), - logits, - topk_config, - ) - return out.topk_ids.long(), out.topk_weights - - def test_routes_like_no_fusion(self): - k = self.TOP_K - for router in self.ROUTERS: - num_experts, _, _, _, biased = router - with self.subTest(router=router): - g = torch.Generator(device="cpu").manual_seed(num_experts) - logits = torch.randn(self.NUM_TOKENS, num_experts, generator=g) - bias = 0.1 * torch.randn(num_experts, generator=g) if biased else None - logits = logits.to(get_device()) - bias = bias.to(get_device()) if biased else None - ref_ids, ref_weights = self._route( - num_fused_shared_experts=0, logits=logits, bias=bias, router=router - ) - ids, weights = self._route( - num_fused_shared_experts=1, logits=logits, bias=bias, router=router - ) - - self.assertEqual(ids.shape, (self.NUM_TOKENS, k + 1)) - torch.testing.assert_close( - weights[:, :k].sum(-1), torch.ones_like(weights[:, 0]) - ) - # Rank r's routed block starts at r * (num_local_routed + 1). - num_local_routed = num_experts // self.EP_SIZE - expected_ids = ref_ids + ref_ids // num_local_routed - order, ref_order = ids[:, :k].argsort(-1), expected_ids.argsort(-1) - self.assertTrue( - torch.equal( - ids[:, :k].gather(-1, order), expected_ids.gather(-1, ref_order) - ) - ) - torch.testing.assert_close( - weights[:, :k].gather(-1, order), ref_weights.gather(-1, ref_order) - ) - home_shared_id = ( - self.EP_RANK * (num_local_routed + 1) + num_local_routed - ) - self.assertTrue(torch.all(ids[:, k] == home_shared_id)) - - def test_unnormalized_global_shared_contribution_is_one(self): - """Raw routed scores must not multiply the global shared contribution.""" - logits = torch.zeros(4, 256, device=get_device()) - rsf = 2.5 - with get_context().override_server_args( - moe_a2a_backend="none", moe_runner_backend="triton" - ): - for folded in (False, True): - with self.subTest(folded=folded): - config = TopKConfig( - top_k=9, - use_grouped_topk=True, - num_expert_group=8, - topk_group=4, - renormalize=False, - num_shared_topk_slots=1, - correction_bias=torch.zeros(256, device=get_device()), - scoring_func="sigmoid", - routed_scaling_factor=rsf, - apply_routed_scaling_factor_on_output=folded, - allow_routed_experts_capture=False, - ) - output = select_experts(logits, logits, config) - torch.testing.assert_close( - output.topk_weights[:, :8], - torch.full_like( - output.topk_weights[:, :8], 0.5 * (rsf if folded else 1.0) - ), - ) - shared = output.topk_weights[:, -1] * (1.0 if folded else rsf) - torch.testing.assert_close(shared, torch.ones_like(shared)) - - def test_unnormalized_weights_stay_raw(self): - """Unnormalized routing must preserve raw selected sigmoid scores.""" - k, num_experts = self.TOP_K, 256 - router = (num_experts, 8, 4, "sigmoid", True) - g = torch.Generator(device="cpu").manual_seed(0) - logits = torch.randn(self.NUM_TOKENS, num_experts, generator=g).to(get_device()) - bias = (0.1 * torch.randn(num_experts, generator=g)).to(get_device()) - _, ref_ids = biased_grouped_topk_impl( - logits, - logits, - bias, - topk=k, - renormalize=False, - num_expert_group=8, - topk_group=4, - ) - num_local_routed = num_experts // self.EP_SIZE - for num_fused_shared_experts in (0, 1): - with self.subTest(num_fused_shared_experts=num_fused_shared_experts): - ids, weights = self._route( - num_fused_shared_experts=num_fused_shared_experts, - logits=logits, - bias=bias, - router=router, - renormalize=False, - ) - routed_ids = ids[:, :k] - if num_fused_shared_experts: - routed_ids = routed_ids - routed_ids // (num_local_routed + 1) - self.assertTrue( - torch.equal( - routed_ids.sort(-1).values, ref_ids.long().sort(-1).values - ) - ) - torch.testing.assert_close( - weights[:, :k], logits.sigmoid().gather(-1, routed_ids) - ) - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/moe/test_hash_topk.py b/test/registered/moe/test_hash_topk.py index 96c46d4d7b..530faf6806 100644 --- a/test/registered/moe/test_hash_topk.py +++ b/test/registered/moe/test_hash_topk.py @@ -4,6 +4,7 @@ from types import SimpleNamespace import pytest import torch +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.moe import hash_topk as hash_topk_module from sglang.srt.layers.moe import topk as topk_module @@ -12,7 +13,7 @@ from sglang.srt.layers.moe.topk import ( StandardTopKOutput, ) from sglang.srt.models.deepseek_v2 import DeepseekV2MoE -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_model, get_parallel from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler from sglang.test.ci.ci_register import register_cpu_ci @@ -42,14 +43,16 @@ def test_hash_topk_remaps_per_rank_fused_shared_slots(monkeypatch, use_aiter): monkeypatch.setattr(get_resources(), "expert_distribution_recorder", FakeRecorder()) - topk = HashTopK( - topk=3, - num_experts=256, - num_fused_shared_experts=1, - vocab_size=2, - scoring_func="sqrtsoftplus", - routed_scaling_factor=2.5, - ) + with get_model().override( + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1) + ): + topk = HashTopK( + topk=2, + num_experts=256, + vocab_size=2, + scoring_func="sqrtsoftplus", + routed_scaling_factor=2.5, + ) with torch.no_grad(): topk.tid2eid.copy_(torch.tensor([[0, 65], [63, 127]], dtype=torch.int32)) @@ -92,13 +95,15 @@ def test_hash_topk_empty_output_keeps_per_rank_shared_slot(monkeypatch): hash_topk_module, "has_per_rank_fused_shared_slots", lambda *_args: True ) - topk = HashTopK( - topk=7, - num_experts=256, - num_fused_shared_experts=1, - vocab_size=2, - scoring_func="softmax", - ) + with get_model().override( + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1) + ): + topk = HashTopK( + topk=6, + num_experts=256, + vocab_size=2, + scoring_func="softmax", + ) output = topk.empty_topk_output(torch.device("cpu")) diff --git a/test/registered/moe/test_shared_expert_routing_extra.py b/test/registered/moe/test_shared_expert_routing_extra.py new file mode 100644 index 0000000000..721fde4611 --- /dev/null +++ b/test/registered/moe/test_shared_expert_routing_extra.py @@ -0,0 +1,215 @@ +"""Extended shared-slot routing and compiled model-context coverage.""" + +import unittest +from unittest.mock import patch + +import torch + +from sglang.srt.configs.shared_experts import SharedExpertsConfig +from sglang.srt.layers.moe import topk as topk_module +from sglang.srt.layers.moe.topk import ( + TopKConfig, + biased_grouped_topk_impl, + select_experts, +) +from sglang.srt.runtime_context import get_context, get_model, get_parallel +from sglang.srt.utils import get_device, is_cuda +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=12, stage="extra-a", runner_config="1-gpu-small") + + +@unittest.skipUnless(is_cuda(), "covers the CUDA routers") +class TestSelectExpertsPerRankSharedSlots(CustomTestCase): + """Shared slots must not alter routed selection or normalization.""" + + def setUp(self): + super().setUp() + override = get_context().override_server_args( + disable_shared_experts_fusion=False + ) + override.install() + self.addCleanup(override.restore) + + NUM_TOKENS, TOP_K, EP_SIZE, EP_RANK = 32, 8, 8, 3 + # (num_experts, num_expert_group, topk_group, scoring_func, biased); each row + # reaches a different router. + ROUTERS = [ + (256, 8, 4, "sigmoid", True), # DeepSeek-V3 grouped routing + (384, 8, 4, "sigmoid", True), # grouped, outside FlashInfer's shapes + (64, 8, 3, "softmax", False), # grouped, no correction bias + (256, None, None, "sqrtsoftplus", True), # DeepSeek-V4 routing + (128, None, None, "softmax", False), + ] + + def _route( + self, *, num_fused_shared_experts, logits, bias, router, renormalize=True + ): + _, num_expert_group, topk_group, scoring_func, _ = router + topk_config = TopKConfig( + top_k=self.TOP_K + num_fused_shared_experts, + renormalize=renormalize, + use_grouped_topk=num_expert_group is not None, + num_expert_group=num_expert_group, + topk_group=topk_group, + shared_experts=SharedExpertsConfig( + num_fused_shared_experts=num_fused_shared_experts + ), + correction_bias=bias, + scoring_func=scoring_func, + routed_scaling_factor=2.5, + allow_routed_experts_capture=False, + ) + with ( + patch.object( + topk_module, + "has_per_rank_fused_shared_slots", + side_effect=lambda num_fused: num_fused > 0, + ), + get_parallel().override(moe_ep_size=self.EP_SIZE, moe_ep_rank=self.EP_RANK), + ): + out = select_experts( + torch.empty(logits.shape[0], 16, device=logits.device), + logits, + topk_config, + ) + return out.topk_ids.long(), out.topk_weights + + def test_routes_like_no_fusion(self): + k = self.TOP_K + for router in self.ROUTERS: + num_experts, _, _, _, biased = router + with self.subTest(router=router): + g = torch.Generator(device="cpu").manual_seed(num_experts) + logits = torch.randn(self.NUM_TOKENS, num_experts, generator=g) + bias = 0.1 * torch.randn(num_experts, generator=g) if biased else None + logits = logits.to(get_device()) + bias = bias.to(get_device()) if biased else None + ref_ids, ref_weights = self._route( + num_fused_shared_experts=0, logits=logits, bias=bias, router=router + ) + ids, weights = self._route( + num_fused_shared_experts=1, logits=logits, bias=bias, router=router + ) + + self.assertEqual(ids.shape, (self.NUM_TOKENS, k + 1)) + torch.testing.assert_close( + weights[:, :k].sum(-1), torch.ones_like(weights[:, 0]) + ) + # Rank r's routed block starts at r * (num_local_routed + 1). + num_local_routed = num_experts // self.EP_SIZE + expected_ids = ref_ids + ref_ids // num_local_routed + order, ref_order = ids[:, :k].argsort(-1), expected_ids.argsort(-1) + self.assertTrue( + torch.equal( + ids[:, :k].gather(-1, order), expected_ids.gather(-1, ref_order) + ) + ) + torch.testing.assert_close( + weights[:, :k].gather(-1, order), ref_weights.gather(-1, ref_order) + ) + home_shared_id = ( + self.EP_RANK * (num_local_routed + 1) + num_local_routed + ) + self.assertTrue(torch.all(ids[:, k] == home_shared_id)) + + def test_unnormalized_global_shared_contribution_is_one(self): + """Raw routed scores must not multiply the global shared contribution.""" + logits = torch.zeros(4, 256, device=get_device()) + rsf = 2.5 + with get_context().override_server_args( + moe_a2a_backend="none", moe_runner_backend="triton" + ): + for folded in (False, True): + with self.subTest(folded=folded): + config = TopKConfig( + top_k=9, + use_grouped_topk=True, + num_expert_group=8, + topk_group=4, + renormalize=False, + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1), + correction_bias=torch.zeros(256, device=get_device()), + scoring_func="sigmoid", + routed_scaling_factor=rsf, + apply_routed_scaling_factor_on_output=folded, + allow_routed_experts_capture=False, + ) + output = select_experts(logits, logits, config) + torch.testing.assert_close( + output.topk_weights[:, :8], + torch.full_like( + output.topk_weights[:, :8], 0.5 * (rsf if folded else 1.0) + ), + ) + shared = output.topk_weights[:, -1] * (1.0 if folded else rsf) + torch.testing.assert_close(shared, torch.ones_like(shared)) + + def test_unnormalized_weights_stay_raw(self): + """Unnormalized routing must preserve raw selected sigmoid scores.""" + k, num_experts = self.TOP_K, 256 + router = (num_experts, 8, 4, "sigmoid", True) + g = torch.Generator(device="cpu").manual_seed(0) + logits = torch.randn(self.NUM_TOKENS, num_experts, generator=g).to(get_device()) + bias = (0.1 * torch.randn(num_experts, generator=g)).to(get_device()) + _, ref_ids = biased_grouped_topk_impl( + logits, + logits, + bias, + topk=k, + renormalize=False, + num_expert_group=8, + topk_group=4, + ) + num_local_routed = num_experts // self.EP_SIZE + for num_fused_shared_experts in (0, 1): + with self.subTest(num_fused_shared_experts=num_fused_shared_experts): + ids, weights = self._route( + num_fused_shared_experts=num_fused_shared_experts, + logits=logits, + bias=bias, + router=router, + renormalize=False, + ) + routed_ids = ids[:, :k] + if num_fused_shared_experts: + routed_ids = routed_ids - routed_ids // (num_local_routed + 1) + self.assertTrue( + torch.equal( + routed_ids.sort(-1).values, ref_ids.long().sort(-1).values + ) + ) + torch.testing.assert_close( + weights[:, :k], logits.sigmoid().gather(-1, routed_ids) + ) + + +class TestSharedExpertsCompiledContext(CustomTestCase): + def setUp(self): + super().setUp() + override = get_context().override_server_args( + disable_shared_experts_fusion=False + ) + override.install() + self.addCleanup(override.restore) + + def test_compile_reads_the_selected_model_config(self): + # Bag leaves are plain attributes: fullgraph tracing must work and must + # distinguish two runners even when the tensor shapes match. + def compute(x): + config = get_model().shared_experts + return x + config.num_fused_shared_experts + config.num_shared_topk_slots + + compiled = torch.compile(compute, backend="eager", fullgraph=True) + for config, expected in ( + (SharedExpertsConfig(num_fused_shared_experts=2), 7), + (SharedExpertsConfig(num_fused_shared_experts=1, use_waterfill=True), 4), + (SharedExpertsConfig(), 3), + ): + with get_model().override(shared_experts=config): + self.assertEqual(compiled(torch.tensor([3])).item(), expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/moe/test_topk_padded_region.py b/test/registered/moe/test_topk_padded_region.py index f53af8b063..af38ae0c44 100644 --- a/test/registered/moe/test_topk_padded_region.py +++ b/test/registered/moe/test_topk_padded_region.py @@ -4,6 +4,7 @@ from unittest.mock import patch import torch import sglang.srt.layers.moe.topk as topk_mod +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.layers.moe.topk import ( TopKConfig, _can_fuse_padded_region, @@ -163,7 +164,9 @@ class TestPostProcessPaddedMaskingHip(CustomTestCase): topk_ids = torch.randint(0, 64, (n, k), device=self.DEVICE, dtype=torch.int32) router_logits = torch.rand((n, 64), device=self.DEVICE, dtype=torch.float32) pad = torch.tensor(n_valid, device=self.DEVICE, dtype=torch.int32) - cfg = TopKConfig(top_k=k, num_shared_topk_slots=0) + cfg = TopKConfig( + top_k=k, shared_experts=SharedExpertsConfig(num_fused_shared_experts=0) + ) _, out_weights, _ = _post_process_topk_ids( topk_ids, topk_weights, @@ -197,7 +200,9 @@ class TestPostProcessPaddedMaskingHip(CustomTestCase): topk_ids = torch.randint(0, 64, (n, k), device=self.DEVICE, dtype=torch.int32) router_logits = torch.rand((n, 64), device=self.DEVICE) pad = torch.tensor(n_valid, device=self.DEVICE, dtype=torch.int32) - cfg = TopKConfig(top_k=k + 1, num_shared_topk_slots=1) + cfg = TopKConfig( + top_k=k + 1, shared_experts=SharedExpertsConfig(num_fused_shared_experts=1) + ) orig_skip = topk_mod._skip_hip_pad_mask orig_aiter = topk_mod._use_aiter topk_mod._skip_hip_pad_mask = True @@ -301,7 +306,7 @@ class TestSelectExpertsPerRankSharedSlotPadMask(CustomTestCase): use_grouped_topk=num_expert_group is not None, num_expert_group=num_expert_group, topk_group=4 if num_expert_group else None, - num_shared_topk_slots=1, + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1), correction_bias=bias.to(self.DEVICE), scoring_func=scoring_func, routed_scaling_factor=2.5, diff --git a/test/registered/moe/test_triton_fused_moe.py b/test/registered/moe/test_triton_fused_moe.py index 9dcb63eedb..0adb5d1c65 100644 --- a/test/registered/moe/test_triton_fused_moe.py +++ b/test/registered/moe/test_triton_fused_moe.py @@ -8,7 +8,7 @@ from sglang.srt.layers.moe import MoeRunner, MoeRunnerBackend, MoeRunnerConfig from sglang.srt.layers.moe.moe_runner.triton_kernels import TritonKernelsQuantInfo from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat -from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.srt.runtime_context import get_context from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -19,6 +19,12 @@ class TestFusedMOE(CustomTestCase): NUM_EXPERTS = [8, 64] TOP_KS = [2, 4] + def setUp(self): + super().setUp() + context_override = get_context().override_server_args() + context_override.install() + self.addCleanup(context_override.restore) + @staticmethod def create_random_cuda_tensor(shape, dtype, mean=0, std=0.01): """Create a random CUDA tensor @@ -59,8 +65,6 @@ class TestFusedMOE(CustomTestCase): topk, return_per_expert: bool = False, ): - set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) - B, D = a.shape a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) diff --git a/test/registered/unit/eplb/test_waterfill_eplb.py b/test/registered/unit/eplb/test_waterfill_eplb.py index 16048ee4bc..7136be0259 100644 --- a/test/registered/unit/eplb/test_waterfill_eplb.py +++ b/test/registered/unit/eplb/test_waterfill_eplb.py @@ -1,5 +1,7 @@ """Unit tests for Waterfill and EPLB updater compatibility.""" +from sglang.srt.configs.shared_experts import SharedExpertsConfig +from sglang.srt.runtime_context import get_model from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -12,7 +14,6 @@ import torch from torch import nn from sglang.srt.layers.moe import topk as topk_module -from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.layers.moe.hash_topk import HashTopK from sglang.srt.layers.moe.topk import TopK, TopKConfig, TopKOutputFormat from sglang.srt.models.deepseek_v2 import DeepseekV2MoE @@ -63,7 +64,7 @@ class TestWaterfillEPLB(CustomTestCase): topk_weights = torch.ones_like(topk_ids, dtype=torch.float32) topk_config = TopKConfig( top_k=4, - num_shared_topk_slots=1, + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1), routed_scaling_factor=1.0, ) dispatch_info = SimpleNamespace( @@ -106,7 +107,7 @@ class TestWaterfillEPLB(CustomTestCase): topk_weights = torch.ones_like(topk_ids, dtype=torch.float32) topk_config = TopKConfig( top_k=4, - num_shared_topk_slots=1, + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1), routed_scaling_factor=1.0, ) dispatch_info = SimpleNamespace( @@ -149,7 +150,7 @@ class TestWaterfillEPLB(CustomTestCase): topk_weights = torch.ones_like(topk_ids, dtype=torch.float32) topk_config = TopKConfig( top_k=4, - num_shared_topk_slots=1, + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1), routed_scaling_factor=1.0, ) dispatch_info = SimpleNamespace( @@ -187,23 +188,25 @@ class TestWaterfillEPLB(CustomTestCase): def test_the_shared_expert_column_has_exactly_one_writer(self): """Shared slots must be written exactly once, by routing or Waterfill.""" + from sglang.srt.runtime_context import get_context + + override = get_context().override_server_args() + override.install() + self.addCleanup(override.restore) + for enable_waterfill in (False, True): - exec_config = SimpleNamespace( - moe=SimpleNamespace(enable_waterfill=enable_waterfill) - ) - with ( - self.subTest(enable_waterfill=enable_waterfill), - patch.object(moe_utils, "get_exec", return_value=exec_config), - ): - topk = TopK(top_k=9, layer_id=0, num_fused_shared_experts=1) - hash_topk = HashTopK( - topk=9, - num_experts=256, - num_fused_shared_experts=1, - vocab_size=4, - layer_id=0, - ) - unfused_topk = TopK(top_k=8, layer_id=0) + with self.subTest(enable_waterfill=enable_waterfill): + with get_model().override( + shared_experts=SharedExpertsConfig( + num_fused_shared_experts=1, use_waterfill=enable_waterfill + ) + ): + topk = TopK(top_k=8, layer_id=0) + hash_topk = HashTopK( + topk=8, num_experts=256, vocab_size=4, layer_id=0 + ) + with get_model().override(shared_experts=SharedExpertsConfig()): + unfused_topk = TopK(top_k=8, layer_id=0) for layer, router_top_k, router_shared_columns in ( ( diff --git a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py index 96e21e863d..771160978e 100644 --- a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py +++ b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py @@ -1,5 +1,7 @@ """Routed and shared contributions must each receive their scale exactly once.""" +from sglang.srt.configs.shared_experts import SharedExpertsConfig +from sglang.srt.runtime_context import get_model from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=15, suite="base-a-test-cpu") @@ -13,7 +15,6 @@ import torch from sglang.srt.batch_overlap.operations import _StateDict from sglang.srt.layers.moe import topk as topk_module -from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.layers.moe.topk import TopK, TopKConfig from sglang.srt.layers.moe.utils import MoeA2ABackend from sglang.srt.model_executor.model_runner_components import moe_ep_setup @@ -42,7 +43,7 @@ class TestFusedSharedExpertScaling(CustomTestCase): topk_config = TopKConfig( top_k=4, - num_shared_topk_slots=1, + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1), routed_scaling_factor=self.ROUTED_SCALING_FACTOR, apply_routed_scaling_factor_on_output=folded_in_topk, **self.ROUTING, @@ -204,6 +205,7 @@ class TestFusedSharedExpertScaling(CustomTestCase): ) module = deepseek_v2_module with ( + get_context().override_server_args(), patch.object(module, "_use_aiter", use_aiter), patch.object(topk_module, "_use_aiter", use_aiter), patch.object( @@ -213,12 +215,10 @@ class TestFusedSharedExpertScaling(CustomTestCase): ), patch.object(module, "get_exec", return_value=exec_config), patch.object(topk_module, "get_exec", return_value=exec_config), - # TopK and HashTopK ask layers/moe/utils.py whether Waterfill is on. - patch.object(moe_utils, "get_exec", return_value=exec_config), - patch.object( - module, - "is_shared_experts_fusion_disabled", - return_value=not fused_shared, + get_model().override( + shared_experts=SharedExpertsConfig( + num_fused_shared_experts=fused_shared + ) ), patch.object(module, "MoEGate", return_value=gate), patch.object( @@ -399,7 +399,7 @@ class TestFusedSharedExpertScaling(CustomTestCase): topk_weights = torch.tensor([[1.0, 0.5, 0.25, 1.0]], dtype=torch.float32) topk_config = TopKConfig( top_k=4, - num_shared_topk_slots=1, + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1), fused_shared_experts_scaling_factor=scaling_factor, allow_routed_experts_capture=False, ) @@ -444,7 +444,7 @@ class TestFusedSharedExpertScaling(CustomTestCase): topk_weights = torch.tensor([[1.0, 0.5, 0.25]], dtype=torch.float32) topk_config = TopKConfig( top_k=4, - num_shared_topk_slots=1, + shared_experts=SharedExpertsConfig(num_fused_shared_experts=1), routed_scaling_factor=self.ROUTED_SCALING_FACTOR, ) with ( @@ -472,7 +472,9 @@ class TestFusedSharedExpertScaling(CustomTestCase): def _route_with_per_rank_slots(self, *, num_fused_shared_experts, logits, bias): topk_config = TopKConfig( top_k=8 + num_fused_shared_experts, - num_shared_topk_slots=num_fused_shared_experts, + shared_experts=SharedExpertsConfig( + num_fused_shared_experts=num_fused_shared_experts + ), renormalize=True, correction_bias=bias, torch_native=True, @@ -529,7 +531,9 @@ class TestFusedSharedExpertScaling(CustomTestCase): here would make the AITER postprocessor append shared ID 288 twice.""" topk = TopK.__new__(TopK) torch.nn.Module.__init__(topk) - topk.topk_config = TopKConfig(top_k=9, num_shared_topk_slots=1) + topk.topk_config = TopKConfig( + top_k=9, shared_experts=SharedExpertsConfig(num_fused_shared_experts=1) + ) topk.enable_waterfill = False topk.waterfill_balancer = None diff --git a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py index c224bf10e9..602cc4ef5c 100644 --- a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py +++ b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py @@ -8,6 +8,7 @@ from unittest.mock import MagicMock, call, patch import torch +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel @@ -454,7 +455,9 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): topk=SimpleNamespace( topk_config=TopKConfig( top_k=2 + fused_shared, - num_shared_topk_slots=fused_shared, + shared_experts=SharedExpertsConfig( + num_fused_shared_experts=fused_shared + ), renormalize=True, routed_scaling_factor=2.0, apply_routed_scaling_factor_on_output=scale_in_topk, diff --git a/test/registered/unit/models/test_deepseek_v4_shared_expert_fusion.py b/test/registered/unit/models/test_deepseek_v4_shared_expert_fusion.py index a93bffd2a6..c8d513b6cd 100644 --- a/test/registered/unit/models/test_deepseek_v4_shared_expert_fusion.py +++ b/test/registered/unit/models/test_deepseek_v4_shared_expert_fusion.py @@ -6,15 +6,22 @@ import torch from torch import nn from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.layers.moe import MoeA2ABackend from sglang.srt.layers.moe.utils import ( - install_shared_experts_fusion_decision, is_shared_experts_fusion_disabled, + resolve_shared_experts_config, ) from sglang.srt.models import deepseek_v4 from sglang.srt.models.deepseek_v4 import DeepseekV4ForCausalLM from sglang.srt.models.deepseek_v4_dspark import DeepseekV4ForCausalLMDSpark -from sglang.srt.runtime_context import get_context, get_exec, get_flags, get_parallel +from sglang.srt.runtime_context import ( + get_context, + get_exec, + get_flags, + get_model, + get_parallel, +) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -22,12 +29,7 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu") class TestDeepseekV4SharedExpertFusionPolicy(CustomTestCase): - """V4 fuses its shared expert only when explicitly asked to. - - The gate is a question the loader asks the model class before any layer - exists (``shared_experts_fusion_disable_reason``); the answer is installed - on the ACTIVE moe flag, and the config bag keeps the user's intent. - """ + """V4 shared-expert fusion requires explicit startup intent.""" def setUp(self): super().setUp() @@ -41,18 +43,18 @@ class TestDeepseekV4SharedExpertFusionPolicy(CustomTestCase): ) override.install() self.addCleanup(override.restore) - get_flags().moe.disable_shared_experts_fusion = None - self.addCleanup( - lambda: setattr(get_flags().moe, "disable_shared_experts_fusion", None) - ) def _install(self, model_class=DeepseekV4ForCausalLM, n_shared_experts=1): - install_shared_experts_fusion_decision( + config = resolve_shared_experts_config( model_class, SimpleNamespace(n_shared_experts=n_shared_experts), None, ) + scope = get_model().override(shared_experts=config) + scope.__enter__() + self.addCleanup(scope.__exit__, None, None, None) + def _make_dspark_config(self): return DeepSeekV4Config( architectures=["DeepseekV4ForCausalLMDSpark"], @@ -77,7 +79,7 @@ class TestDeepseekV4SharedExpertFusionPolicy(CustomTestCase): "Config does not support fused shared expert(s).", ) self._install() - # The decision lands on the ACTIVE flag; the config intent is untouched. + # The resolved model decision leaves the startup intent untouched. self.assertTrue(is_shared_experts_fusion_disabled()) self.assertFalse(get_exec().moe.disable_shared_experts_fusion) @@ -86,6 +88,7 @@ class TestDeepseekV4SharedExpertFusionPolicy(CustomTestCase): with DP attention and CUDA MegaMoE. CP, PP and other A2A paths still lack vision routing support and must fail before constructing the tower. """ + self._publish(enforce=False) config = DeepSeekV4Config( architectures=["DeepseekV4ForCausalLM"], model_type="deepseek_v41", @@ -121,10 +124,12 @@ class TestDeepseekV4SharedExpertFusionPolicy(CustomTestCase): attn_tp_size=2 // cp, attn_cp_size=cp, moe_ep_size=4, + pp_size=pp, pp_group=SimpleNamespace(world_size=pp, is_last_rank=True), ), - get_flags().moe.override( - a2a_backend=backend, disable_shared_experts_fusion=True + get_flags().moe.override(a2a_backend=backend), + get_model().override( + shared_experts=SharedExpertsConfig(fusion_disabled=True) ), patch.object(deepseek_v4, "_is_cuda", cuda), patch.object(deepseek_v4, "ViT", return_value=nn.Identity()), diff --git a/test/registered/unit/models/test_draft_entry_hook_parity.py b/test/registered/unit/models/test_draft_entry_hook_parity.py index d1ebb4ce2e..36c42e03ef 100644 --- a/test/registered/unit/models/test_draft_entry_hook_parity.py +++ b/test/registered/unit/models/test_draft_entry_hook_parity.py @@ -1,19 +1,4 @@ -"""A draft entry class answers the loader's questions exactly when its target does. - -The loader asks the *entry class* it instantiates for the shared-experts-fusion -decision (`install_shared_experts_fusion_decision`). A draft is its own entry -class -- `...NextN`, `...MTP`, `...DSpark`, `...Eagle3` -- so when the target -family carries auto-disable conditions and the draft's class does not expose -them, the draft resolves a *different* decision than the target it drafts for, -and its weights are laid out for the other layout. That shipped once: the DSV4 -DSpark draft skipped its bundled shared-expert tensors until #33312 gave it the -gate. - -This case asks the invariant directly instead: **presence parity between a draft -entry class and its target**. Identity is deliberately not required -- a draft -that delegates with adapted arguments (the Qwen3.5 MTP unwraps `text_config` and -substitutes the MTP quantization config) is exactly right. -""" +"""Draft and target entry classes must expose compatible fusion hooks.""" import re import unittest @@ -28,7 +13,29 @@ register_cpu_ci(est_time=14, suite="base-a-test-cpu") # disagreement is a defect rather than a difference. Weight-name maps # (`packed_modules_mapping`, `hf_to_sglang_mapper`) are deliberately absent: a # draft's checkpoint has its own names, so those differ by design. -PARITY_HOOKS = ("shared_experts_fusion_disable_reason",) +PARITY_HOOKS = ("shared_experts_fusion_disable_reason", "num_fusable_shared_experts") + +# Gated entry classes that declare no count, on purpose: HYV4's gate always +# disables fusion, and the Qwen3.5 family (with the models built on it) fuses a +# gated shared expert inside its own MoE block, outside the loader's count. +GATE_WITHOUT_COUNT = frozenset( + { + "HYV4ForCausalLM", + "HYV4ForCausalLMNextN", + "InternS2MobiusForConditionalGeneration", + "InternS2PreviewForConditionalGeneration", + "MiniCPMV", + "MiniCPMV4_6ForConditionalGeneration", + "MiniCPMV4_7ForConditionalGeneration", + "Qwen3_5ForCausalLM", + "Qwen3_5ForCausalLMMTP", + "Qwen3_5ForConditionalGeneration", + "Qwen3_5MoeForCausalLM", + "Qwen3_5MoeForConditionalGeneration", + "Qwen4ExpForCausalLMMTP", + "Qwen4ExpForConditionalGeneration", + } +) # `...Eagle`/`...Eagle3` drafts are standalone checkpoints with their own # architecture; the rest mirror a stage of their target. @@ -47,14 +54,19 @@ def _target_of(arch: str, archs: dict): return None +def _entry_classes() -> dict: + archs = {} + for arch in sorted(ModelRegistry.get_supported_archs()): + try: + archs[arch] = ModelRegistry.resolve_model_cls(arch)[0] + except Exception: + continue # an arch this environment cannot import + return archs + + class TestDraftEntryHookParity(CustomTestCase): def test_a_draft_resolves_a_gate_exactly_when_its_target_does(self): - archs = {} - for arch in sorted(ModelRegistry.get_supported_archs()): - try: - archs[arch] = ModelRegistry.resolve_model_cls(arch)[0] - except Exception: - continue # an arch this environment cannot import + archs = _entry_classes() checked, offenders = 0, [] for arch, cls in archs.items(): @@ -84,5 +96,33 @@ class TestDraftEntryHookParity(CustomTestCase): ) +class TestFusionHookPairing(CustomTestCase): + def test_a_fusion_gate_comes_with_a_count(self): + archs = _entry_classes() + gated = { + arch + for arch, cls in archs.items() + if hasattr(cls, "shared_experts_fusion_disable_reason") + } + missing = sorted( + arch + for arch in gated - GATE_WITHOUT_COUNT + if not hasattr(archs[arch], "num_fusable_shared_experts") + ) + self.assertEqual( + [], + missing, + "these entry classes expose a fusion gate but no " + "num_fusable_shared_experts, so the loader runs them unfused; declare " + "the count (delegating like the gate), or list the class in " + "GATE_WITHOUT_COUNT with the reason it never fuses routed slots", + ) + self.assertEqual( + [], + sorted(GATE_WITHOUT_COUNT - gated), + "GATE_WITHOUT_COUNT names entry classes that are gone or lost their gate", + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_shared_experts_fusion_gates.py b/test/registered/unit/models/test_shared_experts_fusion_gates.py index 6cde7f7f44..11d92d15f2 100644 --- a/test/registered/unit/models/test_shared_experts_fusion_gates.py +++ b/test/registered/unit/models/test_shared_experts_fusion_gates.py @@ -1,18 +1,4 @@ -"""Every MoE family's fusion gate, asked the way the loader asks it. - -`install_shared_experts_fusion_decision` calls -`.shared_experts_fusion_disable_reason(hf_config, quant_config)` -before the model is built, so the gate must answer from the config and -quantization it is handed — no instance, no layers. These cases pin each -family's branch table, which matters because most of these checkpoints cannot -be run on a single dev box: a wrong answer here is a silently wrong weight -remap (the loader remaps `mlp.shared_experts` into a fused slot the layers -never allocated), not a crash. - -Conditions that depend on the device or the parallel topology are exercised -through `get_parallel().override(...)`; the ones that are pure config / -quantization are exercised directly. -""" +"""Model fusion gates must reject unsupported layouts before construction.""" import importlib.util import sys @@ -733,32 +719,18 @@ class TestBailingMoeV3Gate(_FusionGateCase): self.assertIn("different quant methods", reason) - def test_nextn_constructor_calls_v3_fusion_setup(self): - bailing_moe_v3, bailing_moe_nextn = _import_bailing_modules() + def test_nextn_fuses_the_v3_count_only_for_a_v3_checkpoint(self): + # The NextN entry class builds Ling-V3 layers that read the fused count + # for a V3 checkpoint, and V2.5 layers that never fuse otherwise. + _, bailing_moe_nextn = _import_bailing_modules() + nextn = bailing_moe_nextn.BailingMoeForCausalLMNextN config = SimpleNamespace( - architectures=["BailingMoeForCausalLMNextN"], - model_type="bailing_hybrid", - use_kda=True, - num_shared_experts=1, - vocab_size=32000, - hidden_size=4096, + model_type="bailing_hybrid", use_kda=True, num_shared_experts=1 ) - self._seed(enable_dp_lm_head=False) - with ( - get_parallel().override(tp_size=1, moe_ep_size=1), - unittest.mock.patch.object( - bailing_moe_v3, - "is_shared_experts_fusion_disabled", - return_value=False, - ), - unittest.mock.patch.object(bailing_moe_nextn, "BailingMoEModelNextN"), - unittest.mock.patch.object(bailing_moe_nextn, "ParallelLMHead"), - unittest.mock.patch.object(bailing_moe_nextn, "LogitsProcessor"), - ): - model = bailing_moe_nextn.BailingMoeForCausalLMNextN(config) - - self.assertEqual(model.num_fused_shared_experts, 1) + self.assertEqual(nextn.num_fusable_shared_experts(config), 1) + config.use_kda = False + self.assertEqual(nextn.num_fusable_shared_experts(config), 0) class TestQwen3_5Gate(_FusionGateCase): @@ -791,15 +763,7 @@ class TestQwen3_5Gate(_FusionGateCase): class TestWrapperEntryClassGates(_FusionGateCase): - """A wrapper model answers with the config it hands its nested family. - - The loader asks the class it instantiates, which for these models is the - wrapper — not the DeepSeek/Qwen3.5 body inside it. Each wrapper therefore - delegates to its family's gate with the config (and quantization) the - nested construction uses; these cases pin *what gets handed over*, because - handing over the top-level config instead would answer for the wrong - checkpoint (or raise on a config that has no expert counts at all). - """ + """Wrapper gates must use the config and quantization of their nested model.""" def _recording_gate(self, family_cls): seen = {} @@ -838,6 +802,9 @@ class TestWrapperEntryClassGates(_FusionGateCase): ), "an encoder-only Kimi-VL builds no language model", ) + count = KimiVLForConditionalGeneration.num_fusable_shared_experts + self.assertEqual(count(config), 1) + self.assertEqual(count(SimpleNamespace(encoder_only=True, text_config=None)), 0) def test_kimi_k25_hands_over_its_text_config(self): from sglang.srt.models.deepseek_v2 import DeepseekV3ForCausalLM @@ -870,6 +837,9 @@ class TestWrapperEntryClassGates(_FusionGateCase): SimpleNamespace(encoder_only=True, text_config=None), ) ) + count = KimiK25ForConditionalGeneration.num_fusable_shared_experts + self.assertEqual(count(config), 1) + self.assertEqual(count(SimpleNamespace(encoder_only=True, text_config=None)), 0) def test_pixtral_only_asks_for_its_mla_backbone(self): from sglang.srt.models.mistral_large_3 import MistralLarge3ForCausalLM @@ -890,12 +860,11 @@ class TestWrapperEntryClassGates(_FusionGateCase): ) self.assertIs(seen["config"], mla_text) # A GQA text config builds the dense Mistral backbone instead. - self.assertIsNone( - self._reason( - PixtralForConditionalGeneration, - SimpleNamespace(text_config=SimpleNamespace(model_type="mistral")), - ) - ) + gqa = SimpleNamespace(text_config=SimpleNamespace(model_type="mistral")) + self.assertIsNone(self._reason(PixtralForConditionalGeneration, gqa)) + count = PixtralForConditionalGeneration.num_fusable_shared_experts + self.assertEqual(count(SimpleNamespace(text_config=mla_text)), 1) + self.assertEqual(count(gqa), 0) def test_dots_vlm_hands_over_the_language_config(self): from sglang.srt.models.deepseek_v2 import DeepseekV2ForCausalLM @@ -912,6 +881,11 @@ class TestWrapperEntryClassGates(_FusionGateCase): self._reason(DotsVLMForCausalLM, config, _quant("fp8")) self.assertIs(seen["config"], language_config) self.assertEqual(seen["quant"].get_name(), "fp8") + count = DotsVLMForCausalLM.num_fusable_shared_experts + self.assertEqual(count(config), 1) + self.assertEqual( + count(SimpleNamespace(encoder_only=True, language_config=None)), 0 + ) def test_deepseek_vl2_mirrors_its_unquantized_language_model(self): from sglang.srt.models.deepseek_v2 import DeepseekV2ForCausalLM @@ -935,12 +909,11 @@ class TestWrapperEntryClassGates(_FusionGateCase): seen["quant"], "the language model is constructed without quantization" ) # deepseek-vl2-tiny forbids MLA and builds the dense model instead. - self.assertIsNone( - self._reason( - DeepseekVL2ForCausalLM, - SimpleNamespace(language_config=SimpleNamespace(use_mla=False)), - ) - ) + tiny = SimpleNamespace(language_config=SimpleNamespace(use_mla=False)) + self.assertIsNone(self._reason(DeepseekVL2ForCausalLM, tiny)) + count = DeepseekVL2ForCausalLM.num_fusable_shared_experts + self.assertEqual(count(SimpleNamespace(language_config=language_config)), 1) + self.assertEqual(count(tiny), 0) def test_deepseek_ocr_only_asks_for_its_moe_branches(self): from sglang.srt.models.deepseek_ocr import DeepseekOCRForCausalLM @@ -976,6 +949,10 @@ class TestWrapperEntryClassGates(_FusionGateCase): text_config=SimpleNamespace(topk_method="greedy", use_mla=False), ) self.assertIsNone(self._reason(DeepseekOCRForCausalLM, dense)) + count = DeepseekOCRForCausalLM.num_fusable_shared_experts + self.assertEqual(count(moe_config), 1) + self.assertEqual(count(ocr2), 0) + self.assertEqual(count(dense), 0) def test_minicpmv_entries_delegate_to_the_qwen3_5_gate(self): from sglang.srt.models.minicpmv import ( diff --git a/test/registered/unit/spec/test_draft_construction_isolation.py b/test/registered/unit/spec/test_draft_construction_isolation.py index 85bf84b575..def9b1b8fd 100644 --- a/test/registered/unit/spec/test_draft_construction_isolation.py +++ b/test/registered/unit/spec/test_draft_construction_isolation.py @@ -1,25 +1,18 @@ -"""A draft's construction decides for itself and leaves the process state alone. - -The shared-experts-fusion decision is per checkpoint: each MoE model's gate -writes the ACTIVE moe flag (both ways) before its own layers build and read -it, and ``draft_model_build_scope`` — which brackets every draft -construction — records it on the speculative leaf and restores the target's -value on exit. The config bag keeps the -user's intent. A draft's weight update does not rewrite the -process's model_path record. -""" +"""A model owns its shared-expert layout across construction and execution.""" import unittest -from types import SimpleNamespace +from types import MethodType, SimpleNamespace +from unittest import mock + +import torch from sglang.srt.layers.moe.utils import ( draft_model_build_scope, - install_shared_experts_fusion_decision, - is_shared_experts_fusion_disabled, - speculative_moe_backend_context, + get_num_fused_shared_experts, + resolve_shared_experts_config, ) from sglang.srt.model_executor.model_runner import ModelRunner -from sglang.srt.runtime_context import get_context, get_flags, get_model, publish +from sglang.srt.runtime_context import get_context, get_model from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -38,142 +31,201 @@ class _NoGate: """A model family without an auto-disable gate: it follows the intent.""" -def _install(model_class): - install_shared_experts_fusion_decision(model_class, SimpleNamespace(), None) +class _FusesOne: + """A family that fuses one shared expert as a routed slot unless vetoed.""" + + @staticmethod + def shared_experts_fusion_disable_reason(hf_config, quant_config): + return None + + @staticmethod + def num_fusable_shared_experts(hf_config): + return 1 -class TestFusionDecisionFlag(CustomTestCase): +class _AlwaysDisablesButCountsOne(_FusesOne): + @staticmethod + def shared_experts_fusion_disable_reason(hf_config, quant_config): + return "stand-in: this checkpoint cannot fuse." + + +class _CountsOneWithoutGate: + @staticmethod + def num_fusable_shared_experts(hf_config): + return 1 + + +class TestSharedExpertsModelScope(CustomTestCase): + """Target/draft construction and execution must select the same fixed layout.""" + def setUp(self): super().setUp() - moe = get_flags().moe - self._saved = ( - moe.disable_shared_experts_fusion, - moe.speculative_disable_shared_experts_fusion, + override = get_context().override_server_args( + disable_shared_experts_fusion=False ) - moe.disable_shared_experts_fusion = None - moe.speculative_disable_shared_experts_fusion = None - moe.in_speculative_scope = False - - def tearDown(self): - moe = get_flags().moe - ( - moe.disable_shared_experts_fusion, - moe.speculative_disable_shared_experts_fusion, - ) = self._saved - super().tearDown() - - def _seed(self, **fields): - override = get_context().override_server_args(**fields) override.install() self.addCleanup(override.restore) - def test_unset_flag_falls_back_to_the_config_intent(self): - self._seed(disable_shared_experts_fusion=True) - self.assertTrue(is_shared_experts_fusion_disabled()) - self._seed(disable_shared_experts_fusion=False) - # A fresh install replaces the published config; the flag is still None. - self.assertFalse(is_shared_experts_fusion_disabled()) + def _build(self, count, *, disabled=False, fail=False): + from sglang.srt.layers.moe.topk import TopK + from sglang.srt.model_loader import loader - def test_the_installed_decision_wins_over_the_intent(self): - self._seed(disable_shared_experts_fusion=False) - _install(_AlwaysDisables) - self.assertTrue(is_shared_experts_fusion_disabled()) - _install(_NoGate) - self.assertFalse(is_shared_experts_fusion_disabled()) + class Model(torch.nn.Module): + @staticmethod + def shared_experts_fusion_disable_reason(hf_config, quant_config): + return "checkpoint veto" if hf_config.disabled else None - def test_the_intent_short_circuits_the_gate(self): - # A user who passed --disable-shared-experts-fusion is not overruled, - # and the gate is not even asked. - self._seed(disable_shared_experts_fusion=True) - _install(_NoGate) - self.assertTrue(is_shared_experts_fusion_disabled()) + @staticmethod + def num_fusable_shared_experts(hf_config): + return hf_config.count - def test_the_draft_build_scope_restores_the_targets_decision(self): - self._seed(disable_shared_experts_fusion=False) - _install(_NoGate) # the target's build - with draft_model_build_scope(): - _install(_AlwaysDisables) # the draft's build - self.assertTrue(is_shared_experts_fusion_disabled()) - self.assertFalse(is_shared_experts_fusion_disabled()) - # The draft's decision stays inspectable on the twin leaf. - self.assertTrue(get_flags().moe.speculative_disable_shared_experts_fusion) + def __init__(self, config, quant_config): + super().__init__() + self.topk = TopK(top_k=2, layer_id=0) + self.built_config = get_model().shared_experts + if config.fail: + raise RuntimeError("construction failed") - def test_a_gateless_draft_inherits_the_active_decision(self): - self._seed(disable_shared_experts_fusion=False) - _install(_AlwaysDisables) # the target's build - with draft_model_build_scope(): - # A draft whose family has no gate follows the intent, which is what - # the target's own build already resolved to here. - self.assertTrue(is_shared_experts_fusion_disabled()) - self.assertTrue(is_shared_experts_fusion_disabled()) + def forward_split_prefill(self, input_ids, positions, batch, layer_range): + # Reads both the current model view and the layout retained at build. + return ( + input_ids + + get_num_fused_shared_experts() + + self.topk.topk_config.num_shared_topk_slots + ) - def test_post_build_scopes_do_not_clobber_the_draft_leaf(self): - # init_attention_backends / cuda-graph capture / draft forwards enter - # scopes after construction; no gate runs there, so the persisted - # draft decision must survive. - self._seed(disable_shared_experts_fusion=False) - _install(_NoGate) # target's build - with draft_model_build_scope(): - _install(_AlwaysDisables) # draft's build - for _ in range(3): - with draft_model_build_scope(): - pass - with speculative_moe_backend_context(): - pass - self.assertTrue(get_flags().moe.speculative_disable_shared_experts_fusion) - self.assertFalse(get_flags().moe.disable_shared_experts_fusion) + with mock.patch.object( + loader, "get_model_architecture", return_value=(Model, "Model") + ): + return loader._initialize_model( + SimpleNamespace( + hf_config=SimpleNamespace(count=count, disabled=disabled, fail=fail) + ), + SimpleNamespace(draft_model_idx=None), + ) - def test_the_build_scope_leaves_the_runner_backend_alone(self): - # Swapping runner_backend is speculative_moe_backend_context's job and - # must bracket the draft's whole lifecycle; dflash/dspark run their - # draft outside it, so a construction-only swap would build and - # execute the draft under different backends. - self._seed() - before = get_flags().moe.runner_backend - with draft_model_build_scope(): - self.assertEqual(get_flags().moe.runner_backend, before) - self.assertEqual(get_flags().moe.runner_backend, before) + def _runner(self, model): + runner = ModelRunner.__new__(ModelRunner) + runner.model = model + runner.model_config = SimpleNamespace(num_hidden_layers=2) + runner.device_timer = None + return runner - def test_a_record_outside_any_scope_is_target_only(self): - self._seed(disable_shared_experts_fusion=False) - get_flags().moe.speculative_disable_shared_experts_fusion = True - _install(_NoGate) # target's build - self.assertTrue(get_flags().moe.speculative_disable_shared_experts_fusion) - - def test_initialize_moe_config_seeds_both_leaves(self): - from sglang.srt.layers.moe.utils import initialize_moe_config - from sglang.srt.server_args import ServerArgs - - self._seed() - publish( - ServerArgs(model_path="dummy", disable_shared_experts_fusion=True), - role="scheduler", + def _run(self, runner): + batch = SimpleNamespace( + split_index=1, input_ids=torch.tensor([3]), positions=None ) - initialize_moe_config() - moe = get_flags().moe - self.assertTrue(moe.disable_shared_experts_fusion) - self.assertTrue(moe.speculative_disable_shared_experts_fusion) + return runner.forward_split_prefill(batch) - def test_a_forward_time_read_is_refused(self): - # The invariant behind the whole design: the decision is consumed at - # construction only. During a draft's build the flag holds the draft's - # value, so a forward reading it would race the build window. - from sglang.srt.model_executor.forward_context import ( - ForwardContext, - forward_context, - ) + def test_loader_and_alternating_runners_keep_their_own_layout(self): + ambient = get_model().shared_experts + target = self._build(2) + with draft_model_build_scope(): + draft = self._build(1, disabled=True) + self.assertIs(get_model().shared_experts, ambient) + for model in (target, draft): + self.assertIs(model.shared_experts_config, model.built_config) + self.assertIs( + model.shared_experts_config, model.topk.topk_config.shared_experts + ) + target_runner, draft_runner = self._runner(target), self._runner(draft) + for runner, expected in ( + (target_runner, 7), + (draft_runner, 3), + (target_runner, 7), + ): + self.assertEqual(self._run(runner).item(), expected) + self.assertIs(get_model().shared_experts, ambient) + # A model replacement must not leave the runner reading its old config. + target_runner.model = draft + self.assertEqual(self._run(target_runner).item(), 3) - self._seed() - with forward_context(ForwardContext(attn_backend=SimpleNamespace())): - with self.assertRaises(AssertionError): - is_shared_experts_fusion_disabled() + def test_nested_build_and_execution_restore_on_failure(self): + target = self._build(1) + with get_model().override(shared_experts=target.shared_experts_config): + with self.assertRaisesRegex(RuntimeError, "construction failed"): + self._build(2, fail=True) + self.assertIs(get_model().shared_experts, target.shared_experts_config) + draft = self._build(1, disabled=True) + runner = self._runner(draft) + with mock.patch.object( + draft, + "forward_split_prefill", + side_effect=RuntimeError("forward failed"), + ): + with self.assertRaisesRegex(RuntimeError, "forward failed"): + self._run(runner) + self.assertIs(get_model().shared_experts, target.shared_experts_config) + self.assertEqual(self._run(self._runner(target)).item(), 5) - def test_the_intent_stays_on_the_bag(self): - self._seed(disable_shared_experts_fusion=False) - _install(_AlwaysDisables) + def test_resolution_preserves_intent_and_requires_both_hooks(self): from sglang.srt.runtime_context import get_exec + ambient = get_model().shared_experts + for cls, disabled, count in ( + (_NoGate, False, 0), + (_CountsOneWithoutGate, False, 0), + (_AlwaysDisablesButCountsOne, True, 0), + (_FusesOne, False, 1), + ): + with self.subTest(cls=cls.__name__): + config = resolve_shared_experts_config(cls, SimpleNamespace(), None) + self.assertEqual( + (config.fusion_disabled, config.num_fused_shared_experts), + (disabled, count), + ) + with get_exec().moe.override(disable_shared_experts_fusion=True): + config = resolve_shared_experts_config(_FusesOne, SimpleNamespace(), None) + self.assertTrue(config.fusion_disabled) + self.assertEqual(config.num_fused_shared_experts, 0) + with get_exec().moe.override(enable_waterfill=True): + fused = resolve_shared_experts_config(_FusesOne, SimpleNamespace(), None) + unfused = resolve_shared_experts_config( + _AlwaysDisablesButCountsOne, SimpleNamespace(), None + ) + self.assertEqual( + (fused.num_fused_shared_experts, fused.num_shared_topk_slots), (1, 0) + ) + self.assertTrue(fused.use_waterfill) + self.assertFalse(unfused.use_waterfill) self.assertFalse(get_exec().moe.disable_shared_experts_fusion) + self.assertIs(get_model().shared_experts, ambient) + + def test_warmup_and_direct_capture_bind_the_model_config(self): + from sglang.srt.model_executor.cpu_graph_runner import CPUGraphRunner + from sglang.srt.model_executor.runner.base_runner import BaseRunner + from sglang.srt.model_executor.runner.decode_cuda_graph_runner import ( + DecodeCudaGraphRunner, + ) + from sglang.srt.model_executor.runner.prefill_cuda_graph_runner import ( + PrefillCudaGraphRunner, + ) + + model = self._build(1) + ambient = get_model().shared_experts + for entry, body in ( + (BaseRunner.warmup, "_warmup"), + (DecodeCudaGraphRunner.capture, "_capture"), + (PrefillCudaGraphRunner.capture, "_capture"), + (CPUGraphRunner.capture, "_capture"), + ): + observed = [] + + def capture(): + observed.append(get_model().shared_experts) + raise RuntimeError("capture failed") + + graph_runner = SimpleNamespace( + model_runner=self._runner(model), **{body: capture} + ) + graph_runner.model_context = MethodType( + BaseRunner.model_context, graph_runner + ) + with self.subTest(entry=entry.__qualname__): + with self.assertRaisesRegex(RuntimeError, "capture failed"): + entry(graph_runner) + self.assertEqual(observed, [model.shared_experts_config]) + self.assertIs(get_model().shared_experts, ambient) class TestDraftWeightUpdateRecord(CustomTestCase): diff --git a/test/registered/unit/state_capturer/test_routed_experts_scattered_a2a.py b/test/registered/unit/state_capturer/test_routed_experts_scattered_a2a.py index 8ff6aa3e58..9ad5065c87 100644 --- a/test/registered/unit/state_capturer/test_routed_experts_scattered_a2a.py +++ b/test/registered/unit/state_capturer/test_routed_experts_scattered_a2a.py @@ -6,6 +6,7 @@ from unittest import mock import torch +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.layers.moe.utils import MoeA2ABackend from sglang.srt.state_capturer import routed_experts as re_mod from sglang.srt.state_capturer.base import BaseDeviceCache @@ -118,5 +119,34 @@ class TestCaptureRoutedIdsOnly(CustomTestCase): self._capture(self._capturer(), ids, "none") +class TestCapturerWidthFollowsTheInstalledCount(CustomTestCase): + def test_a_wrapper_model_gets_room_for_the_fused_shared_column(self): + """Capture sizing must use the supplied wrapper model's shared-expert layout.""" + seen = {} + + def record(self, model_config, **kwargs): + seen.update(kwargs) + + features = SimpleNamespace(enable_return_routed_experts=True) + with ( + mock.patch.object( + re_mod, "get_exec", return_value=SimpleNamespace(features=features) + ), + mock.patch.object(re_mod.RoutedExpertsCapturer, "__init__", record), + ): + re_mod.RoutedExpertsCapturer.create( + model=SimpleNamespace( + shared_experts_config=SharedExpertsConfig( + num_fused_shared_experts=1 + ) + ), + model_config=SimpleNamespace(), + num_tokens=8, + max_running_requests=2, + device="cpu", + ) + self.assertEqual(seen["num_fused_shared_experts"], 1) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/xpu/test_topk.py b/test/registered/xpu/test_topk.py index 26b2f7ce10..93fcd49297 100644 --- a/test/registered/xpu/test_topk.py +++ b/test/registered/xpu/test_topk.py @@ -3,6 +3,7 @@ from typing import Optional import torch +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.layers.moe.hash_topk import HashTopK torch.use_deterministic_algorithms(True) @@ -21,7 +22,7 @@ from sglang.srt.layers.moe.topk import grouped_topk_gpu as native_grouped_topk from sglang.srt.layers.moe.topk import ( grouped_topk_xpu, ) -from sglang.srt.runtime_context import get_context +from sglang.srt.runtime_context import get_context, get_model from sglang.test.ci.ci_register import register_xpu_ci from sglang.test.test_utils import CustomTestCase @@ -380,14 +381,18 @@ class TestBiasedGroupedTopK(CustomTestCase): device=device, ) - hash_topk = HashTopK( - topk=topk, - num_experts=E_num, - num_fused_shared_experts=num_fused_shared_experts, - vocab_size=vocab_size, - scoring_func="sqrtsoftplus", - routed_scaling_factor=2.5, - ).to(device) + with get_model().override( + shared_experts=SharedExpertsConfig( + num_fused_shared_experts=num_fused_shared_experts + ) + ): + hash_topk = HashTopK( + topk=topk - num_fused_shared_experts, + num_experts=E_num, + vocab_size=vocab_size, + scoring_func="sqrtsoftplus", + routed_scaling_factor=2.5, + ).to(device) topk_routed = hash_topk.tid2eid.shape[1] with torch.no_grad(): hash_topk.tid2eid.copy_(