[Fix] [Apple] mlx: Give the stub model a shared-expert layout (#45419)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: R0CKSTAR <yeahdongcn@gmail.com>
This commit is contained in:
Leo
2026-10-21 15:54:49 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 R0CKSTAR
parent f4035cab0b
commit ed1f56fead
2 changed files with 29 additions and 0 deletions
@@ -10,6 +10,7 @@ from typing import Tuple
import torch
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.configs.shared_experts import SharedExpertsConfig
from sglang.srt.hardware_backend.mlx.kv_cache.auxiliary_state import (
MlxAuxiliaryStateReqToTokenPool,
)
@@ -84,6 +85,10 @@ class _DummyModel:
"""Minimal stand-in so that `inspect.signature(model.forward)` and
`getattr(model, ...)` calls in ModelRunner.__init__ don't crash."""
# The model loader sets this on every Torch model, and the graph runners
# read it during warmup. No Torch MoE layers exist here, so nothing is fused.
shared_experts_config = SharedExpertsConfig()
@staticmethod
def forward():
pass
@@ -11,6 +11,9 @@ attention backend named by ``server_args.attention_backend``; MLX never
uses one, and some backends read real KV buffers in ``__init__``, which
crashes on ``_DummyKVCache``.
The graph runners select ``model.shared_experts_config`` during warmup. The
Torch model loader sets it, so the stub's placeholder model must carry one.
The base ``preloaded_weights_bytes`` property reads the Torch model loader.
The MLX stub never creates one because its native runner owns model loading
and KV sizing, so it must report zero for the Torch accounting hook.
@@ -39,6 +42,7 @@ if _HAS_MLX:
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.model_executor.runner.base_runner import BaseRunner
@unittest.skipUnless(_HAS_MLX, _SKIP_REASON)
@@ -85,6 +89,26 @@ class TestMlxRunnerPoolContract(CustomTestCase):
runner.account_preloaded_weights(runner.preloaded_weights_bytes)
self.assertEqual(runner.pre_model_load_memory, 1.0)
def test_stub_model_supports_runner_model_context(self):
"""Runner warmup must find a shared-expert layout on the stub model."""
runner = object.__new__(MlxModelRunnerStub)
runner.model_config = SimpleNamespace(
is_hybrid_swa=False,
sliding_window_size=None,
attention_chunk_size=None,
dtype=None,
)
runner.load_model()
graph_runner = SimpleNamespace(model_runner=runner)
with (
get_context().override_server_args(),
BaseRunner.model_context(graph_runner),
):
pass
self.assertEqual(runner.model.shared_experts_config.num_fused_shared_experts, 0)
def test_stub_overrides_base_alloc_memory_pool(self):
self.assertIn(
"alloc_memory_pool",